1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
  25  * Copyright 2016 RackTop Systems.
  26  */
  27 
  28 /*
  29  * graph.c - master restarter graph engine
  30  *
  31  *   The graph engine keeps a dependency graph of all service instances on the
  32  *   system, as recorded in the repository.  It decides when services should
  33  *   be brought up or down based on service states and dependencies and sends
  34  *   commands to restarters to effect any changes.  It also executes
  35  *   administrator commands sent by svcadm via the repository.
  36  *
  37  *   The graph is stored in uu_list_t *dgraph and its vertices are
  38  *   graph_vertex_t's, each of which has a name and an integer id unique to
  39  *   its name (see dict.c).  A vertex's type attribute designates the type
  40  *   of object it represents: GVT_INST for service instances, GVT_SVC for
  41  *   service objects (since service instances may depend on another service,
  42  *   rather than service instance), GVT_FILE for files (which services may
  43  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
  44  *   vertices are necessary because dependency lists may have particular
  45  *   grouping types (require any, require all, optional, or exclude) and
  46  *   event-propagation characteristics.
  47  *
  48  *   The initial graph is built by libscf_populate_graph() invoking
  49  *   dgraph_add_instance() for each instance in the repository.  The function
  50  *   adds a GVT_SVC vertex for the service if one does not already exist, adds
  51  *   a GVT_INST vertex named by the FMRI of the instance, and sets up the edges.
  52  *   The resulting web of vertices & edges associated with an instance's vertex
  53  *   includes
  54  *
  55  *     - an edge from the GVT_SVC vertex for the instance's service
  56  *
  57  *     - an edge to the GVT_INST vertex of the instance's resarter, if its
  58  *       restarter is not svc.startd
  59  *
  60  *     - edges from other GVT_INST vertices if the instance is a restarter
  61  *
  62  *     - for each dependency property group in the instance's "running"
  63  *       snapshot, an edge to a GVT_GROUP vertex named by the FMRI of the
  64  *       instance and the name of the property group
  65  *
  66  *     - for each value of the "entities" property in each dependency property
  67  *       group, an edge from the corresponding GVT_GROUP vertex to a
  68  *       GVT_INST, GVT_SVC, or GVT_FILE vertex
  69  *
  70  *     - edges from GVT_GROUP vertices for each dependent instance
  71  *
  72  *   After the edges are set up the vertex's GV_CONFIGURED flag is set.  If
  73  *   there are problems, or if a service is mentioned in a dependency but does
  74  *   not exist in the repository, the GV_CONFIGURED flag will be clear.
  75  *
  76  *   The graph and all of its vertices are protected by the dgraph_lock mutex.
  77  *   See restarter.c for more information.
  78  *
  79  *   The properties of an instance fall into two classes: immediate and
  80  *   snapshotted.  Immediate properties should have an immediate effect when
  81  *   changed.  Snapshotted properties should be read from a snapshot, so they
  82  *   only change when the snapshot changes.  The immediate properties used by
  83  *   the graph engine are general/enabled, general/restarter, and the properties
  84  *   in the restarter_actions property group.  Since they are immediate, they
  85  *   are not read out of a snapshot.  The snapshotted properties used by the
  86  *   graph engine are those in the property groups with type "dependency" and
  87  *   are read out of the "running" snapshot.  The "running" snapshot is created
  88  *   by the the graph engine as soon as possible, and it is updated, along with
  89  *   in-core copies of the data (dependency information for the graph engine) on
  90  *   receipt of the refresh command from svcadm.  In addition, the graph engine
  91  *   updates the "start" snapshot from the "running" snapshot whenever a service
  92  *   comes online.
  93  *
  94  *   When a DISABLE event is requested by the administrator, svc.startd shutdown
  95  *   the dependents first before shutting down the requested service.
  96  *   In graph_enable_by_vertex, we create a subtree that contains the dependent
  97  *   vertices by marking those vertices with the GV_TOOFFLINE flag. And we mark
  98  *   the vertex to disable with the GV_TODISABLE flag. Once the tree is created,
  99  *   we send the _ADMIN_DISABLE event to the leaves. The leaves will then
 100  *   transition from STATE_ONLINE/STATE_DEGRADED to STATE_OFFLINE/STATE_MAINT.
 101  *   In gt_enter_offline and gt_enter_maint if the vertex was in a subtree then
 102  *   we clear the GV_TOOFFLINE flag and walk the dependencies to offline the new
 103  *   exposed leaves. We do the same until we reach the last leaf (the one with
 104  *   the GV_TODISABLE flag). If the vertex to disable is also part of a larger
 105  *   subtree (eg. multiple DISABLE events on vertices in the same subtree) then
 106  *   once the first vertex is disabled (GV_TODISABLE flag is removed), we
 107  *   continue to propagate the offline event to the vertex's dependencies.
 108  *
 109  *
 110  * SMF state transition notifications
 111  *
 112  *   When an instance of a service managed by SMF changes state, svc.startd may
 113  *   publish a GPEC sysevent. All transitions to or from maintenance, a
 114  *   transition cause by a hardware error will generate an event.
 115  *   Other transitions will generate an event if there exist notification
 116  *   parameter for that transition. Notification parameters are stored in the
 117  *   SMF repository for the service/instance they refer to. System-wide
 118  *   notification parameters are stored in the global instance.
 119  *   svc.startd can be told to send events for all SMF state transitions despite
 120  *   of notification parameters by setting options/info_events_all to true in
 121  *   restarter:default
 122  *
 123  *   The set of transitions that generate events is cached in the
 124  *   dgraph_vertex_t gv_stn_tset for service/instance and in the global
 125  *   stn_global for the system-wide set. They are re-read when instances are
 126  *   refreshed.
 127  *
 128  *   The GPEC events published by svc.startd are consumed by fmd(1M). After
 129  *   processing these events, fmd(1M) publishes the processed events to
 130  *   notification agents. The notification agents read the notification
 131  *   parameters from the SMF repository through libscf(3LIB) interfaces and send
 132  *   the notification, or not, based on those parameters.
 133  *
 134  *   Subscription and publishing to the GPEC channels is done with the
 135  *   libfmevent(3LIB) wrappers fmev_[r]publish_*() and
 136  *   fmev_shdl_(un)subscribe().
 137  *
 138  */
 139 
 140 #include <sys/uadmin.h>
 141 #include <sys/wait.h>
 142 
 143 #include <assert.h>
 144 #include <errno.h>
 145 #include <fcntl.h>
 146 #include <fm/libfmevent.h>
 147 #include <libscf.h>
 148 #include <libscf_priv.h>
 149 #include <librestart.h>
 150 #include <libuutil.h>
 151 #include <locale.h>
 152 #include <poll.h>
 153 #include <pthread.h>
 154 #include <signal.h>
 155 #include <stddef.h>
 156 #include <stdio.h>
 157 #include <stdlib.h>
 158 #include <string.h>
 159 #include <strings.h>
 160 #include <sys/statvfs.h>
 161 #include <sys/uadmin.h>
 162 #include <zone.h>
 163 #if defined(__i386)
 164 #include <libgrubmgmt.h>
 165 #endif  /* __i386 */
 166 
 167 #include "startd.h"
 168 #include "protocol.h"
 169 
 170 
 171 #define MILESTONE_NONE  ((graph_vertex_t *)1)
 172 
 173 #define CONSOLE_LOGIN_FMRI      "svc:/system/console-login:default"
 174 #define FS_MINIMAL_FMRI         "svc:/system/filesystem/minimal:default"
 175 
 176 #define VERTEX_REMOVED  0       /* vertex has been freed  */
 177 #define VERTEX_INUSE    1       /* vertex is still in use */
 178 
 179 #define IS_ENABLED(v) ((v)->gv_flags & (GV_ENABLED | GV_ENBLD_NOOVR))
 180 
 181 /*
 182  * stn_global holds the tset for the system wide notification parameters.
 183  * It is updated on refresh of svc:/system/svc/global:default
 184  *
 185  * There are two assumptions that relax the need for a mutex:
 186  *     1. 32-bit value assignments are atomic
 187  *     2. Its value is consumed only in one point at
 188  *     dgraph_state_transition_notify(). There are no test and set races.
 189  *
 190  *     If either assumption is broken, we'll need a mutex to synchronize
 191  *     access to stn_global
 192  */
 193 int32_t stn_global;
 194 /*
 195  * info_events_all holds a flag to override notification parameters and send
 196  * Information events for all state transitions.
 197  * same about the need of a mutex here.
 198  */
 199 int info_events_all;
 200 
 201 /*
 202  * Services in these states are not considered 'down' by the
 203  * milestone/shutdown code.
 204  */
 205 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
 206         (state) == RESTARTER_STATE_DEGRADED || \
 207         (state) == RESTARTER_STATE_OFFLINE)
 208 
 209 #define is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
 210         ((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \
 211         (v->gv_restart < RERR_RESTART)))
 212 
 213 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
 214 static uu_list_t *dgraph;
 215 static pthread_mutex_t dgraph_lock;
 216 
 217 /*
 218  * milestone indicates the current subgraph.  When NULL, it is the entire
 219  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
 220  * services on which the target vertex depends.
 221  */
 222 static graph_vertex_t *milestone = NULL;
 223 static boolean_t initial_milestone_set = B_FALSE;
 224 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
 225 
 226 /* protected by dgraph_lock */
 227 static boolean_t sulogin_thread_running = B_FALSE;
 228 static boolean_t sulogin_running = B_FALSE;
 229 static boolean_t console_login_ready = B_FALSE;
 230 
 231 /* Number of services to come down to complete milestone transition. */
 232 static uint_t non_subgraph_svcs;
 233 
 234 /*
 235  * These variables indicate what should be done when we reach the milestone
 236  * target milestone, i.e., when non_subgraph_svcs == 0.  They are acted upon in
 237  * dgraph_set_instance_state().
 238  */
 239 static int halting = -1;
 240 static boolean_t go_single_user_mode = B_FALSE;
 241 static boolean_t go_to_level1 = B_FALSE;
 242 
 243 /*
 244  * Tracks when we started halting.
 245  */
 246 static time_t halting_time = 0;
 247 
 248 /*
 249  * This tracks the legacy runlevel to ensure we signal init and manage
 250  * utmpx entries correctly.
 251  */
 252 static char current_runlevel = '\0';
 253 
 254 /* Number of single user threads currently running */
 255 static pthread_mutex_t single_user_thread_lock;
 256 static int single_user_thread_count = 0;
 257 
 258 /* Statistics for dependency cycle-checking */
 259 static u_longlong_t dep_inserts = 0;
 260 static u_longlong_t dep_cycle_ns = 0;
 261 static u_longlong_t dep_insert_ns = 0;
 262 
 263 
 264 static const char * const emsg_invalid_restarter =
 265         "Transitioning %s to maintenance, restarter FMRI %s is invalid "
 266         "(see 'svcs -xv' for details).\n";
 267 static const char * const console_login_fmri = CONSOLE_LOGIN_FMRI;
 268 static const char * const single_user_fmri = SCF_MILESTONE_SINGLE_USER;
 269 static const char * const multi_user_fmri = SCF_MILESTONE_MULTI_USER;
 270 static const char * const multi_user_svr_fmri = SCF_MILESTONE_MULTI_USER_SERVER;
 271 
 272 
 273 /*
 274  * These services define the system being "up".  If none of them can come
 275  * online, then we will run sulogin on the console.  Note that the install ones
 276  * are for the miniroot and when installing CDs after the first.  can_come_up()
 277  * does the decision making, and an sulogin_thread() runs sulogin, which can be
 278  * started by dgraph_set_instance_state() or single_user_thread().
 279  *
 280  * NOTE: can_come_up() relies on SCF_MILESTONE_SINGLE_USER being the first
 281  * entry, which is only used when booting_to_single_user (boot -s) is set.
 282  * This is because when doing a "boot -s", sulogin is started from specials.c
 283  * after milestone/single-user comes online, for backwards compatibility.
 284  * In this case, SCF_MILESTONE_SINGLE_USER needs to be part of up_svcs
 285  * to ensure sulogin will be spawned if milestone/single-user cannot be reached.
 286  */
 287 static const char * const up_svcs[] = {
 288         SCF_MILESTONE_SINGLE_USER,
 289         CONSOLE_LOGIN_FMRI,
 290         "svc:/system/install-setup:default",
 291         "svc:/system/install:default",
 292         NULL
 293 };
 294 
 295 /* This array must have an element for each non-NULL element of up_svcs[]. */
 296 static graph_vertex_t *up_svcs_p[] = { NULL, NULL, NULL, NULL };
 297 
 298 /* These are for seed repository magic.  See can_come_up(). */
 299 static const char * const manifest_import = SCF_INSTANCE_MI;
 300 static graph_vertex_t *manifest_import_p = NULL;
 301 
 302 
 303 static char target_milestone_as_runlevel(void);
 304 static void graph_runlevel_changed(char rl, int online);
 305 static int dgraph_set_milestone(const char *, scf_handle_t *, boolean_t);
 306 static boolean_t should_be_in_subgraph(graph_vertex_t *v);
 307 static int mark_subtree(graph_edge_t *, void *);
 308 static boolean_t insubtree_dependents_down(graph_vertex_t *);
 309 
 310 /*
 311  * graph_vertex_compare()
 312  *      This function can compare either int *id or * graph_vertex_t *gv
 313  *      values, as the vertex id is always the first element of a
 314  *      graph_vertex structure.
 315  */
 316 /* ARGSUSED */
 317 static int
 318 graph_vertex_compare(const void *lc_arg, const void *rc_arg, void *private)
 319 {
 320         int lc_id = ((const graph_vertex_t *)lc_arg)->gv_id;
 321         int rc_id = *(int *)rc_arg;
 322 
 323         if (lc_id > rc_id)
 324                 return (1);
 325         if (lc_id < rc_id)
 326                 return (-1);
 327         return (0);
 328 }
 329 
 330 void
 331 graph_init()
 332 {
 333         graph_edge_pool = startd_list_pool_create("graph_edges",
 334             sizeof (graph_edge_t), offsetof(graph_edge_t, ge_link), NULL,
 335             UU_LIST_POOL_DEBUG);
 336         assert(graph_edge_pool != NULL);
 337 
 338         graph_vertex_pool = startd_list_pool_create("graph_vertices",
 339             sizeof (graph_vertex_t), offsetof(graph_vertex_t, gv_link),
 340             graph_vertex_compare, UU_LIST_POOL_DEBUG);
 341         assert(graph_vertex_pool != NULL);
 342 
 343         (void) pthread_mutex_init(&dgraph_lock, &mutex_attrs);
 344         (void) pthread_mutex_init(&single_user_thread_lock, &mutex_attrs);
 345         dgraph = startd_list_create(graph_vertex_pool, NULL, UU_LIST_SORTED);
 346         assert(dgraph != NULL);
 347 
 348         if (!st->st_initial)
 349                 current_runlevel = utmpx_get_runlevel();
 350 
 351         log_framework(LOG_DEBUG, "Initialized graph\n");
 352 }
 353 
 354 static graph_vertex_t *
 355 vertex_get_by_name(const char *name)
 356 {
 357         int id;
 358 
 359         assert(MUTEX_HELD(&dgraph_lock));
 360 
 361         id = dict_lookup_byname(name);
 362         if (id == -1)
 363                 return (NULL);
 364 
 365         return (uu_list_find(dgraph, &id, NULL, NULL));
 366 }
 367 
 368 static graph_vertex_t *
 369 vertex_get_by_id(int id)
 370 {
 371         assert(MUTEX_HELD(&dgraph_lock));
 372 
 373         if (id == -1)
 374                 return (NULL);
 375 
 376         return (uu_list_find(dgraph, &id, NULL, NULL));
 377 }
 378 
 379 /*
 380  * Creates a new vertex with the given name, adds it to the graph, and returns
 381  * a pointer to it.  The graph lock must be held by this thread on entry.
 382  */
 383 static graph_vertex_t *
 384 graph_add_vertex(const char *name)
 385 {
 386         int id;
 387         graph_vertex_t *v;
 388         void *p;
 389         uu_list_index_t idx;
 390 
 391         assert(MUTEX_HELD(&dgraph_lock));
 392 
 393         id = dict_insert(name);
 394 
 395         v = startd_zalloc(sizeof (*v));
 396 
 397         v->gv_id = id;
 398 
 399         v->gv_name = startd_alloc(strlen(name) + 1);
 400         (void) strcpy(v->gv_name, name);
 401 
 402         v->gv_dependencies = startd_list_create(graph_edge_pool, v, 0);
 403         v->gv_dependents = startd_list_create(graph_edge_pool, v, 0);
 404 
 405         p = uu_list_find(dgraph, &id, NULL, &idx);
 406         assert(p == NULL);
 407 
 408         uu_list_node_init(v, &v->gv_link, graph_vertex_pool);
 409         uu_list_insert(dgraph, v, idx);
 410 
 411         return (v);
 412 }
 413 
 414 /*
 415  * Removes v from the graph and frees it.  The graph should be locked by this
 416  * thread, and v should have no edges associated with it.
 417  */
 418 static void
 419 graph_remove_vertex(graph_vertex_t *v)
 420 {
 421         assert(MUTEX_HELD(&dgraph_lock));
 422 
 423         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 424         assert(uu_list_numnodes(v->gv_dependents) == 0);
 425         assert(v->gv_refs == 0);
 426 
 427         startd_free(v->gv_name, strlen(v->gv_name) + 1);
 428         uu_list_destroy(v->gv_dependencies);
 429         uu_list_destroy(v->gv_dependents);
 430         uu_list_remove(dgraph, v);
 431 
 432         startd_free(v, sizeof (graph_vertex_t));
 433 }
 434 
 435 static void
 436 graph_add_edge(graph_vertex_t *fv, graph_vertex_t *tv)
 437 {
 438         graph_edge_t *e, *re;
 439         int r;
 440 
 441         assert(MUTEX_HELD(&dgraph_lock));
 442 
 443         e = startd_alloc(sizeof (graph_edge_t));
 444         re = startd_alloc(sizeof (graph_edge_t));
 445 
 446         e->ge_parent = fv;
 447         e->ge_vertex = tv;
 448 
 449         re->ge_parent = tv;
 450         re->ge_vertex = fv;
 451 
 452         uu_list_node_init(e, &e->ge_link, graph_edge_pool);
 453         r = uu_list_insert_before(fv->gv_dependencies, NULL, e);
 454         assert(r == 0);
 455 
 456         uu_list_node_init(re, &re->ge_link, graph_edge_pool);
 457         r = uu_list_insert_before(tv->gv_dependents, NULL, re);
 458         assert(r == 0);
 459 }
 460 
 461 static void
 462 graph_remove_edge(graph_vertex_t *v, graph_vertex_t *dv)
 463 {
 464         graph_edge_t *e;
 465 
 466         for (e = uu_list_first(v->gv_dependencies);
 467             e != NULL;
 468             e = uu_list_next(v->gv_dependencies, e)) {
 469                 if (e->ge_vertex == dv) {
 470                         uu_list_remove(v->gv_dependencies, e);
 471                         startd_free(e, sizeof (graph_edge_t));
 472                         break;
 473                 }
 474         }
 475 
 476         for (e = uu_list_first(dv->gv_dependents);
 477             e != NULL;
 478             e = uu_list_next(dv->gv_dependents, e)) {
 479                 if (e->ge_vertex == v) {
 480                         uu_list_remove(dv->gv_dependents, e);
 481                         startd_free(e, sizeof (graph_edge_t));
 482                         break;
 483                 }
 484         }
 485 }
 486 
 487 static void
 488 remove_inst_vertex(graph_vertex_t *v)
 489 {
 490         graph_edge_t *e;
 491         graph_vertex_t *sv;
 492         int i;
 493 
 494         assert(MUTEX_HELD(&dgraph_lock));
 495         assert(uu_list_numnodes(v->gv_dependents) == 1);
 496         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 497         assert(v->gv_refs == 0);
 498         assert((v->gv_flags & GV_CONFIGURED) == 0);
 499 
 500         e = uu_list_first(v->gv_dependents);
 501         sv = e->ge_vertex;
 502         graph_remove_edge(sv, v);
 503 
 504         for (i = 0; up_svcs[i] != NULL; ++i) {
 505                 if (up_svcs_p[i] == v)
 506                         up_svcs_p[i] = NULL;
 507         }
 508 
 509         if (manifest_import_p == v)
 510                 manifest_import_p = NULL;
 511 
 512         graph_remove_vertex(v);
 513 
 514         if (uu_list_numnodes(sv->gv_dependencies) == 0 &&
 515             uu_list_numnodes(sv->gv_dependents) == 0 &&
 516             sv->gv_refs == 0)
 517                 graph_remove_vertex(sv);
 518 }
 519 
 520 static void
 521 graph_walk_dependents(graph_vertex_t *v, void (*func)(graph_vertex_t *, void *),
 522     void *arg)
 523 {
 524         graph_edge_t *e;
 525 
 526         for (e = uu_list_first(v->gv_dependents);
 527             e != NULL;
 528             e = uu_list_next(v->gv_dependents, e))
 529                 func(e->ge_vertex, arg);
 530 }
 531 
 532 static void
 533 graph_walk_dependencies(graph_vertex_t *v, void (*func)(graph_vertex_t *,
 534         void *), void *arg)
 535 {
 536         graph_edge_t *e;
 537 
 538         assert(MUTEX_HELD(&dgraph_lock));
 539 
 540         for (e = uu_list_first(v->gv_dependencies);
 541             e != NULL;
 542             e = uu_list_next(v->gv_dependencies, e)) {
 543 
 544                 func(e->ge_vertex, arg);
 545         }
 546 }
 547 
 548 /*
 549  * Generic graph walking function.
 550  *
 551  * Given a vertex, this function will walk either dependencies
 552  * (WALK_DEPENDENCIES) or dependents (WALK_DEPENDENTS) of a vertex recursively
 553  * for the entire graph.  It will avoid cycles and never visit the same vertex
 554  * twice.
 555  *
 556  * We avoid traversing exclusion dependencies, because they are allowed to
 557  * create cycles in the graph.  When propagating satisfiability, there is no
 558  * need to walk exclusion dependencies because exclude_all_satisfied() doesn't
 559  * test for satisfiability.
 560  *
 561  * The walker takes two callbacks.  The first is called before examining the
 562  * dependents of each vertex.  The second is called on each vertex after
 563  * examining its dependents.  This allows is_path_to() to construct a path only
 564  * after the target vertex has been found.
 565  */
 566 typedef enum {
 567         WALK_DEPENDENTS,
 568         WALK_DEPENDENCIES
 569 } graph_walk_dir_t;
 570 
 571 typedef int (*graph_walk_cb_t)(graph_vertex_t *, void *);
 572 
 573 typedef struct graph_walk_info {
 574         graph_walk_dir_t        gi_dir;
 575         uchar_t                 *gi_visited;    /* vertex bitmap */
 576         int                     (*gi_pre)(graph_vertex_t *, void *);
 577         void                    (*gi_post)(graph_vertex_t *, void *);
 578         void                    *gi_arg;        /* callback arg */
 579         int                     gi_ret;         /* return value */
 580 } graph_walk_info_t;
 581 
 582 static int
 583 graph_walk_recurse(graph_edge_t *e, graph_walk_info_t *gip)
 584 {
 585         uu_list_t *list;
 586         int r;
 587         graph_vertex_t *v = e->ge_vertex;
 588         int i;
 589         uint_t b;
 590 
 591         i = v->gv_id / 8;
 592         b = 1 << (v->gv_id % 8);
 593 
 594         /*
 595          * Check to see if we've visited this vertex already.
 596          */
 597         if (gip->gi_visited[i] & b)
 598                 return (UU_WALK_NEXT);
 599 
 600         gip->gi_visited[i] |= b;
 601 
 602         /*
 603          * Don't follow exclusions.
 604          */
 605         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
 606                 return (UU_WALK_NEXT);
 607 
 608         /*
 609          * Call pre-visit callback.  If this doesn't terminate the walk,
 610          * continue search.
 611          */
 612         if ((gip->gi_ret = gip->gi_pre(v, gip->gi_arg)) == UU_WALK_NEXT) {
 613                 /*
 614                  * Recurse using appropriate list.
 615                  */
 616                 if (gip->gi_dir == WALK_DEPENDENTS)
 617                         list = v->gv_dependents;
 618                 else
 619                         list = v->gv_dependencies;
 620 
 621                 r = uu_list_walk(list, (uu_walk_fn_t *)graph_walk_recurse,
 622                     gip, 0);
 623                 assert(r == 0);
 624         }
 625 
 626         /*
 627          * Callbacks must return either UU_WALK_NEXT or UU_WALK_DONE.
 628          */
 629         assert(gip->gi_ret == UU_WALK_NEXT || gip->gi_ret == UU_WALK_DONE);
 630 
 631         /*
 632          * If given a post-callback, call the function for every vertex.
 633          */
 634         if (gip->gi_post != NULL)
 635                 (void) gip->gi_post(v, gip->gi_arg);
 636 
 637         /*
 638          * Preserve the callback's return value.  If the callback returns
 639          * UU_WALK_DONE, then we propagate that to the caller in order to
 640          * terminate the walk.
 641          */
 642         return (gip->gi_ret);
 643 }
 644 
 645 static void
 646 graph_walk(graph_vertex_t *v, graph_walk_dir_t dir,
 647     int (*pre)(graph_vertex_t *, void *),
 648     void (*post)(graph_vertex_t *, void *), void *arg)
 649 {
 650         graph_walk_info_t gi;
 651         graph_edge_t fake;
 652         size_t sz = dictionary->dict_new_id / 8 + 1;
 653 
 654         gi.gi_visited = startd_zalloc(sz);
 655         gi.gi_pre = pre;
 656         gi.gi_post = post;
 657         gi.gi_arg = arg;
 658         gi.gi_dir = dir;
 659         gi.gi_ret = 0;
 660 
 661         /*
 662          * Fake up an edge for the first iteration
 663          */
 664         fake.ge_vertex = v;
 665         (void) graph_walk_recurse(&fake, &gi);
 666 
 667         startd_free(gi.gi_visited, sz);
 668 }
 669 
 670 typedef struct child_search {
 671         int     id;             /* id of vertex to look for */
 672         uint_t  depth;          /* recursion depth */
 673         /*
 674          * While the vertex is not found, path is NULL.  After the search, if
 675          * the vertex was found then path should point to a -1-terminated
 676          * array of vertex id's which constitute the path to the vertex.
 677          */
 678         int     *path;
 679 } child_search_t;
 680 
 681 static int
 682 child_pre(graph_vertex_t *v, void *arg)
 683 {
 684         child_search_t *cs = arg;
 685 
 686         cs->depth++;
 687 
 688         if (v->gv_id == cs->id) {
 689                 cs->path = startd_alloc((cs->depth + 1) * sizeof (int));
 690                 cs->path[cs->depth] = -1;
 691                 return (UU_WALK_DONE);
 692         }
 693 
 694         return (UU_WALK_NEXT);
 695 }
 696 
 697 static void
 698 child_post(graph_vertex_t *v, void *arg)
 699 {
 700         child_search_t *cs = arg;
 701 
 702         cs->depth--;
 703 
 704         if (cs->path != NULL)
 705                 cs->path[cs->depth] = v->gv_id;
 706 }
 707 
 708 /*
 709  * Look for a path from from to to.  If one exists, returns a pointer to
 710  * a NULL-terminated array of pointers to the vertices along the path.  If
 711  * there is no path, returns NULL.
 712  */
 713 static int *
 714 is_path_to(graph_vertex_t *from, graph_vertex_t *to)
 715 {
 716         child_search_t cs;
 717 
 718         cs.id = to->gv_id;
 719         cs.depth = 0;
 720         cs.path = NULL;
 721 
 722         graph_walk(from, WALK_DEPENDENCIES, child_pre, child_post, &cs);
 723 
 724         return (cs.path);
 725 }
 726 
 727 /*
 728  * Given an array of int's as returned by is_path_to, allocates a string of
 729  * their names joined by newlines.  Returns the size of the allocated buffer
 730  * in *sz and frees path.
 731  */
 732 static void
 733 path_to_str(int *path, char **cpp, size_t *sz)
 734 {
 735         int i;
 736         graph_vertex_t *v;
 737         size_t allocd, new_allocd;
 738         char *new, *name;
 739 
 740         assert(MUTEX_HELD(&dgraph_lock));
 741         assert(path[0] != -1);
 742 
 743         allocd = 1;
 744         *cpp = startd_alloc(1);
 745         (*cpp)[0] = '\0';
 746 
 747         for (i = 0; path[i] != -1; ++i) {
 748                 name = NULL;
 749 
 750                 v = vertex_get_by_id(path[i]);
 751 
 752                 if (v == NULL)
 753                         name = "<deleted>";
 754                 else if (v->gv_type == GVT_INST || v->gv_type == GVT_SVC)
 755                         name = v->gv_name;
 756 
 757                 if (name != NULL) {
 758                         new_allocd = allocd + strlen(name) + 1;
 759                         new = startd_alloc(new_allocd);
 760                         (void) strcpy(new, *cpp);
 761                         (void) strcat(new, name);
 762                         (void) strcat(new, "\n");
 763 
 764                         startd_free(*cpp, allocd);
 765 
 766                         *cpp = new;
 767                         allocd = new_allocd;
 768                 }
 769         }
 770 
 771         startd_free(path, sizeof (int) * (i + 1));
 772 
 773         *sz = allocd;
 774 }
 775 
 776 
 777 /*
 778  * This function along with run_sulogin() implements an exclusion relationship
 779  * between system/console-login and sulogin.  run_sulogin() will fail if
 780  * system/console-login is online, and the graph engine should call
 781  * graph_clogin_start() to bring system/console-login online, which defers the
 782  * start if sulogin is running.
 783  */
 784 static void
 785 graph_clogin_start(graph_vertex_t *v)
 786 {
 787         assert(MUTEX_HELD(&dgraph_lock));
 788 
 789         if (sulogin_running)
 790                 console_login_ready = B_TRUE;
 791         else
 792                 vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 793 }
 794 
 795 static void
 796 graph_su_start(graph_vertex_t *v)
 797 {
 798         /*
 799          * /etc/inittab used to have the initial /sbin/rcS as a 'sysinit'
 800          * entry with a runlevel of 'S', before jumping to the final
 801          * target runlevel (as set in initdefault).  We mimic that legacy
 802          * behavior here.
 803          */
 804         utmpx_set_runlevel('S', '0', B_FALSE);
 805         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 806 }
 807 
 808 static void
 809 graph_post_su_online(void)
 810 {
 811         graph_runlevel_changed('S', 1);
 812 }
 813 
 814 static void
 815 graph_post_su_disable(void)
 816 {
 817         graph_runlevel_changed('S', 0);
 818 }
 819 
 820 static void
 821 graph_post_mu_online(void)
 822 {
 823         graph_runlevel_changed('2', 1);
 824 }
 825 
 826 static void
 827 graph_post_mu_disable(void)
 828 {
 829         graph_runlevel_changed('2', 0);
 830 }
 831 
 832 static void
 833 graph_post_mus_online(void)
 834 {
 835         graph_runlevel_changed('3', 1);
 836 }
 837 
 838 static void
 839 graph_post_mus_disable(void)
 840 {
 841         graph_runlevel_changed('3', 0);
 842 }
 843 
 844 static struct special_vertex_info {
 845         const char      *name;
 846         void            (*start_f)(graph_vertex_t *);
 847         void            (*post_online_f)(void);
 848         void            (*post_disable_f)(void);
 849 } special_vertices[] = {
 850         { CONSOLE_LOGIN_FMRI, graph_clogin_start, NULL, NULL },
 851         { SCF_MILESTONE_SINGLE_USER, graph_su_start,
 852             graph_post_su_online, graph_post_su_disable },
 853         { SCF_MILESTONE_MULTI_USER, NULL,
 854             graph_post_mu_online, graph_post_mu_disable },
 855         { SCF_MILESTONE_MULTI_USER_SERVER, NULL,
 856             graph_post_mus_online, graph_post_mus_disable },
 857         { NULL },
 858 };
 859 
 860 
 861 void
 862 vertex_send_event(graph_vertex_t *v, restarter_event_type_t e)
 863 {
 864         switch (e) {
 865         case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
 866                 assert(v->gv_state == RESTARTER_STATE_UNINIT);
 867 
 868                 MUTEX_LOCK(&st->st_load_lock);
 869                 st->st_load_instances++;
 870                 MUTEX_UNLOCK(&st->st_load_lock);
 871                 break;
 872 
 873         case RESTARTER_EVENT_TYPE_ENABLE:
 874                 log_framework(LOG_DEBUG, "Enabling %s.\n", v->gv_name);
 875                 assert(v->gv_state == RESTARTER_STATE_UNINIT ||
 876                     v->gv_state == RESTARTER_STATE_DISABLED ||
 877                     v->gv_state == RESTARTER_STATE_MAINT);
 878                 break;
 879 
 880         case RESTARTER_EVENT_TYPE_DISABLE:
 881         case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
 882                 log_framework(LOG_DEBUG, "Disabling %s.\n", v->gv_name);
 883                 assert(v->gv_state != RESTARTER_STATE_DISABLED);
 884                 break;
 885 
 886         case RESTARTER_EVENT_TYPE_STOP_RESET:
 887         case RESTARTER_EVENT_TYPE_STOP:
 888                 log_framework(LOG_DEBUG, "Stopping %s.\n", v->gv_name);
 889                 assert(v->gv_state == RESTARTER_STATE_DEGRADED ||
 890                     v->gv_state == RESTARTER_STATE_ONLINE);
 891                 break;
 892 
 893         case RESTARTER_EVENT_TYPE_START:
 894                 log_framework(LOG_DEBUG, "Starting %s.\n", v->gv_name);
 895                 assert(v->gv_state == RESTARTER_STATE_OFFLINE);
 896                 break;
 897 
 898         case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
 899         case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
 900         case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
 901         case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
 902         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
 903         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
 904         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
 905         case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
 906         case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
 907                 break;
 908 
 909         default:
 910 #ifndef NDEBUG
 911                 uu_warn("%s:%d: Bad event %d.\n", __FILE__, __LINE__, e);
 912 #endif
 913                 abort();
 914         }
 915 
 916         restarter_protocol_send_event(v->gv_name, v->gv_restarter_channel, e,
 917             v->gv_reason);
 918 }
 919 
 920 static void
 921 graph_unset_restarter(graph_vertex_t *v)
 922 {
 923         assert(MUTEX_HELD(&dgraph_lock));
 924         assert(v->gv_flags & GV_CONFIGURED);
 925 
 926         vertex_send_event(v, RESTARTER_EVENT_TYPE_REMOVE_INSTANCE);
 927 
 928         if (v->gv_restarter_id != -1) {
 929                 graph_vertex_t *rv;
 930 
 931                 rv = vertex_get_by_id(v->gv_restarter_id);
 932                 graph_remove_edge(v, rv);
 933         }
 934 
 935         v->gv_restarter_id = -1;
 936         v->gv_restarter_channel = NULL;
 937 }
 938 
 939 /*
 940  * Return VERTEX_REMOVED when the vertex passed in argument is deleted from the
 941  * dgraph otherwise return VERTEX_INUSE.
 942  */
 943 static int
 944 free_if_unrefed(graph_vertex_t *v)
 945 {
 946         assert(MUTEX_HELD(&dgraph_lock));
 947 
 948         if (v->gv_refs > 0)
 949                 return (VERTEX_INUSE);
 950 
 951         if (v->gv_type == GVT_SVC &&
 952             uu_list_numnodes(v->gv_dependents) == 0 &&
 953             uu_list_numnodes(v->gv_dependencies) == 0) {
 954                 graph_remove_vertex(v);
 955                 return (VERTEX_REMOVED);
 956         } else if (v->gv_type == GVT_INST &&
 957             (v->gv_flags & GV_CONFIGURED) == 0 &&
 958             uu_list_numnodes(v->gv_dependents) == 1 &&
 959             uu_list_numnodes(v->gv_dependencies) == 0) {
 960                 remove_inst_vertex(v);
 961                 return (VERTEX_REMOVED);
 962         }
 963 
 964         return (VERTEX_INUSE);
 965 }
 966 
 967 static void
 968 delete_depgroup(graph_vertex_t *v)
 969 {
 970         graph_edge_t *e;
 971         graph_vertex_t *dv;
 972 
 973         assert(MUTEX_HELD(&dgraph_lock));
 974         assert(v->gv_type == GVT_GROUP);
 975         assert(uu_list_numnodes(v->gv_dependents) == 0);
 976 
 977         while ((e = uu_list_first(v->gv_dependencies)) != NULL) {
 978                 dv = e->ge_vertex;
 979 
 980                 graph_remove_edge(v, dv);
 981 
 982                 switch (dv->gv_type) {
 983                 case GVT_INST:          /* instance dependency */
 984                 case GVT_SVC:           /* service dependency */
 985                         (void) free_if_unrefed(dv);
 986                         break;
 987 
 988                 case GVT_FILE:          /* file dependency */
 989                         assert(uu_list_numnodes(dv->gv_dependencies) == 0);
 990                         if (uu_list_numnodes(dv->gv_dependents) == 0)
 991                                 graph_remove_vertex(dv);
 992                         break;
 993 
 994                 default:
 995 #ifndef NDEBUG
 996                         uu_warn("%s:%d: Unexpected node type %d", __FILE__,
 997                             __LINE__, dv->gv_type);
 998 #endif
 999                         abort();
1000                 }
1001         }
1002 
1003         graph_remove_vertex(v);
1004 }
1005 
1006 static int
1007 delete_instance_deps_cb(graph_edge_t *e, void **ptrs)
1008 {
1009         graph_vertex_t *v = ptrs[0];
1010         boolean_t delete_restarter_dep = (boolean_t)ptrs[1];
1011         graph_vertex_t *dv;
1012 
1013         dv = e->ge_vertex;
1014 
1015         /*
1016          * We have four possibilities here:
1017          *   - GVT_INST: restarter
1018          *   - GVT_GROUP - GVT_INST: instance dependency
1019          *   - GVT_GROUP - GVT_SVC - GV_INST: service dependency
1020          *   - GVT_GROUP - GVT_FILE: file dependency
1021          */
1022         switch (dv->gv_type) {
1023         case GVT_INST:  /* restarter */
1024                 assert(dv->gv_id == v->gv_restarter_id);
1025                 if (delete_restarter_dep)
1026                         graph_remove_edge(v, dv);
1027                 break;
1028 
1029         case GVT_GROUP: /* pg dependency */
1030                 graph_remove_edge(v, dv);
1031                 delete_depgroup(dv);
1032                 break;
1033 
1034         case GVT_FILE:
1035                 /* These are currently not direct dependencies */
1036 
1037         default:
1038 #ifndef NDEBUG
1039                 uu_warn("%s:%d: Bad vertex type %d.\n", __FILE__, __LINE__,
1040                     dv->gv_type);
1041 #endif
1042                 abort();
1043         }
1044 
1045         return (UU_WALK_NEXT);
1046 }
1047 
1048 static void
1049 delete_instance_dependencies(graph_vertex_t *v, boolean_t delete_restarter_dep)
1050 {
1051         void *ptrs[2];
1052         int r;
1053 
1054         assert(MUTEX_HELD(&dgraph_lock));
1055         assert(v->gv_type == GVT_INST);
1056 
1057         ptrs[0] = v;
1058         ptrs[1] = (void *)delete_restarter_dep;
1059 
1060         r = uu_list_walk(v->gv_dependencies,
1061             (uu_walk_fn_t *)delete_instance_deps_cb, &ptrs, UU_WALK_ROBUST);
1062         assert(r == 0);
1063 }
1064 
1065 /*
1066  * int graph_insert_vertex_unconfigured()
1067  *   Insert a vertex without sending any restarter events. If the vertex
1068  *   already exists or creation is successful, return a pointer to it in *vp.
1069  *
1070  *   If type is not GVT_GROUP, dt can remain unset.
1071  *
1072  *   Returns 0, EEXIST, or EINVAL if the arguments are invalid (i.e., fmri
1073  *   doesn't agree with type, or type doesn't agree with dt).
1074  */
1075 static int
1076 graph_insert_vertex_unconfigured(const char *fmri, gv_type_t type,
1077     depgroup_type_t dt, restarter_error_t rt, graph_vertex_t **vp)
1078 {
1079         int r;
1080         int i;
1081 
1082         assert(MUTEX_HELD(&dgraph_lock));
1083 
1084         switch (type) {
1085         case GVT_SVC:
1086         case GVT_INST:
1087                 if (strncmp(fmri, "svc:", sizeof ("svc:") - 1) != 0)
1088                         return (EINVAL);
1089                 break;
1090 
1091         case GVT_FILE:
1092                 if (strncmp(fmri, "file:", sizeof ("file:") - 1) != 0)
1093                         return (EINVAL);
1094                 break;
1095 
1096         case GVT_GROUP:
1097                 if (dt <= 0 || rt < 0)
1098                         return (EINVAL);
1099                 break;
1100 
1101         default:
1102 #ifndef NDEBUG
1103                 uu_warn("%s:%d: Unknown type %d.\n", __FILE__, __LINE__, type);
1104 #endif
1105                 abort();
1106         }
1107 
1108         *vp = vertex_get_by_name(fmri);
1109         if (*vp != NULL)
1110                 return (EEXIST);
1111 
1112         *vp = graph_add_vertex(fmri);
1113 
1114         (*vp)->gv_type = type;
1115         (*vp)->gv_depgroup = dt;
1116         (*vp)->gv_restart = rt;
1117 
1118         (*vp)->gv_flags = 0;
1119         (*vp)->gv_state = RESTARTER_STATE_NONE;
1120 
1121         for (i = 0; special_vertices[i].name != NULL; ++i) {
1122                 if (strcmp(fmri, special_vertices[i].name) == 0) {
1123                         (*vp)->gv_start_f = special_vertices[i].start_f;
1124                         (*vp)->gv_post_online_f =
1125                             special_vertices[i].post_online_f;
1126                         (*vp)->gv_post_disable_f =
1127                             special_vertices[i].post_disable_f;
1128                         break;
1129                 }
1130         }
1131 
1132         (*vp)->gv_restarter_id = -1;
1133         (*vp)->gv_restarter_channel = 0;
1134 
1135         if (type == GVT_INST) {
1136                 char *sfmri;
1137                 graph_vertex_t *sv;
1138 
1139                 sfmri = inst_fmri_to_svc_fmri(fmri);
1140                 sv = vertex_get_by_name(sfmri);
1141                 if (sv == NULL) {
1142                         r = graph_insert_vertex_unconfigured(sfmri, GVT_SVC, 0,
1143                             0, &sv);
1144                         assert(r == 0);
1145                 }
1146                 startd_free(sfmri, max_scf_fmri_size);
1147 
1148                 graph_add_edge(sv, *vp);
1149         }
1150 
1151         /*
1152          * If this vertex is in the subgraph, mark it as so, for both
1153          * GVT_INST and GVT_SERVICE verteces.
1154          * A GVT_SERVICE vertex can only be in the subgraph if another instance
1155          * depends on it, in which case it's already been added to the graph
1156          * and marked as in the subgraph (by refresh_vertex()).  If a
1157          * GVT_SERVICE vertex was freshly added (by the code above), it means
1158          * that it has no dependents, and cannot be in the subgraph.
1159          * Regardless of this, we still check that gv_flags includes
1160          * GV_INSUBGRAPH in the event that future behavior causes the above
1161          * code to add a GVT_SERVICE vertex which should be in the subgraph.
1162          */
1163 
1164         (*vp)->gv_flags |= (should_be_in_subgraph(*vp)? GV_INSUBGRAPH : 0);
1165 
1166         return (0);
1167 }
1168 
1169 /*
1170  * Returns 0 on success or ELOOP if the dependency would create a cycle.
1171  */
1172 static int
1173 graph_insert_dependency(graph_vertex_t *fv, graph_vertex_t *tv, int **pathp)
1174 {
1175         hrtime_t now;
1176 
1177         assert(MUTEX_HELD(&dgraph_lock));
1178 
1179         /* cycle detection */
1180         now = gethrtime();
1181 
1182         /* Don't follow exclusions. */
1183         if (!(fv->gv_type == GVT_GROUP &&
1184             fv->gv_depgroup == DEPGRP_EXCLUDE_ALL)) {
1185                 *pathp = is_path_to(tv, fv);
1186                 if (*pathp)
1187                         return (ELOOP);
1188         }
1189 
1190         dep_cycle_ns += gethrtime() - now;
1191         ++dep_inserts;
1192         now = gethrtime();
1193 
1194         graph_add_edge(fv, tv);
1195 
1196         dep_insert_ns += gethrtime() - now;
1197 
1198         /* Check if the dependency adds the "to" vertex to the subgraph */
1199         tv->gv_flags |= (should_be_in_subgraph(tv) ? GV_INSUBGRAPH : 0);
1200 
1201         return (0);
1202 }
1203 
1204 static int
1205 inst_running(graph_vertex_t *v)
1206 {
1207         assert(v->gv_type == GVT_INST);
1208 
1209         if (v->gv_state == RESTARTER_STATE_ONLINE ||
1210             v->gv_state == RESTARTER_STATE_DEGRADED)
1211                 return (1);
1212 
1213         return (0);
1214 }
1215 
1216 /*
1217  * The dependency evaluation functions return
1218  *   1 - dependency satisfied
1219  *   0 - dependency unsatisfied
1220  *   -1 - dependency unsatisfiable (without administrator intervention)
1221  *
1222  * The functions also take a boolean satbility argument.  When true, the
1223  * functions may recurse in order to determine satisfiability.
1224  */
1225 static int require_any_satisfied(graph_vertex_t *, boolean_t);
1226 static int dependency_satisfied(graph_vertex_t *, boolean_t);
1227 
1228 /*
1229  * A require_all dependency is unsatisfied if any elements are unsatisfied.  It
1230  * is unsatisfiable if any elements are unsatisfiable.
1231  */
1232 static int
1233 require_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1234 {
1235         graph_edge_t *edge;
1236         int i;
1237         boolean_t any_unsatisfied;
1238 
1239         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1240                 return (1);
1241 
1242         any_unsatisfied = B_FALSE;
1243 
1244         for (edge = uu_list_first(groupv->gv_dependencies);
1245             edge != NULL;
1246             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1247                 i = dependency_satisfied(edge->ge_vertex, satbility);
1248                 if (i == 1)
1249                         continue;
1250 
1251                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1252                     "require_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1253                     edge->ge_vertex->gv_name, i == 0 ? "ed" : "able");
1254 
1255                 if (!satbility)
1256                         return (0);
1257 
1258                 if (i == -1)
1259                         return (-1);
1260 
1261                 any_unsatisfied = B_TRUE;
1262         }
1263 
1264         return (any_unsatisfied ? 0 : 1);
1265 }
1266 
1267 /*
1268  * A require_any dependency is satisfied if any element is satisfied.  It is
1269  * satisfiable if any element is satisfiable.
1270  */
1271 static int
1272 require_any_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1273 {
1274         graph_edge_t *edge;
1275         int s;
1276         boolean_t satisfiable;
1277 
1278         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1279                 return (1);
1280 
1281         satisfiable = B_FALSE;
1282 
1283         for (edge = uu_list_first(groupv->gv_dependencies);
1284             edge != NULL;
1285             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1286                 s = dependency_satisfied(edge->ge_vertex, satbility);
1287 
1288                 if (s == 1)
1289                         return (1);
1290 
1291                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1292                     "require_any(%s): %s is unsatisfi%s.\n",
1293                     groupv->gv_name, edge->ge_vertex->gv_name,
1294                     s == 0 ? "ed" : "able");
1295 
1296                 if (satbility && s == 0)
1297                         satisfiable = B_TRUE;
1298         }
1299 
1300         return (!satbility || satisfiable ? 0 : -1);
1301 }
1302 
1303 /*
1304  * An optional_all dependency only considers elements which are configured,
1305  * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1306  * is unsatisfied.
1307  *
1308  * Offline dependencies which are waiting for a dependency to come online are
1309  * unsatisfied.  Offline dependences which cannot possibly come online
1310  * (unsatisfiable) are always considered satisfied.
1311  */
1312 static int
1313 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1314 {
1315         graph_edge_t *edge;
1316         graph_vertex_t *v;
1317         boolean_t any_qualified;
1318         boolean_t any_unsatisfied;
1319         int i;
1320 
1321         any_qualified = B_FALSE;
1322         any_unsatisfied = B_FALSE;
1323 
1324         for (edge = uu_list_first(groupv->gv_dependencies);
1325             edge != NULL;
1326             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1327                 v = edge->ge_vertex;
1328 
1329                 switch (v->gv_type) {
1330                 case GVT_INST:
1331                         /* Skip missing instances */
1332                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1333                                 continue;
1334 
1335                         if (v->gv_state == RESTARTER_STATE_MAINT)
1336                                 continue;
1337 
1338                         any_qualified = B_TRUE;
1339                         if (v->gv_state == RESTARTER_STATE_OFFLINE) {
1340                                 /*
1341                                  * For offline dependencies, treat unsatisfiable
1342                                  * as satisfied.
1343                                  */
1344                                 i = dependency_satisfied(v, B_TRUE);
1345                                 if (i == -1)
1346                                         i = 1;
1347                         } else if (v->gv_state == RESTARTER_STATE_DISABLED) {
1348                                 /*
1349                                  * The service is enabled, but hasn't
1350                                  * transitioned out of disabled yet.  Treat it
1351                                  * as unsatisfied (not unsatisfiable).
1352                                  */
1353                                 i = v->gv_flags & GV_ENABLED ? 0 : 1;
1354                         } else {
1355                                 i = dependency_satisfied(v, satbility);
1356                         }
1357                         break;
1358 
1359                 case GVT_FILE:
1360                         any_qualified = B_TRUE;
1361                         i = dependency_satisfied(v, satbility);
1362 
1363                         break;
1364 
1365                 case GVT_SVC: {
1366                         boolean_t svc_any_qualified;
1367                         boolean_t svc_satisfied;
1368                         boolean_t svc_satisfiable;
1369                         graph_vertex_t *v2;
1370                         graph_edge_t *e2;
1371 
1372                         svc_any_qualified = B_FALSE;
1373                         svc_satisfied = B_FALSE;
1374                         svc_satisfiable = B_FALSE;
1375 
1376                         for (e2 = uu_list_first(v->gv_dependencies);
1377                             e2 != NULL;
1378                             e2 = uu_list_next(v->gv_dependencies, e2)) {
1379                                 v2 = e2->ge_vertex;
1380                                 assert(v2->gv_type == GVT_INST);
1381 
1382                                 if ((v2->gv_flags & GV_CONFIGURED) == 0)
1383                                         continue;
1384 
1385                                 if (v2->gv_state == RESTARTER_STATE_MAINT)
1386                                         continue;
1387 
1388                                 svc_any_qualified = B_TRUE;
1389 
1390                                 if (v2->gv_state == RESTARTER_STATE_OFFLINE) {
1391                                         /*
1392                                          * For offline dependencies, treat
1393                                          * unsatisfiable as satisfied.
1394                                          */
1395                                         i = dependency_satisfied(v2, B_TRUE);
1396                                         if (i == -1)
1397                                                 i = 1;
1398                                 } else if (v2->gv_state ==
1399                                     RESTARTER_STATE_DISABLED) {
1400                                         i = v2->gv_flags & GV_ENABLED ? 0 : 1;
1401                                 } else {
1402                                         i = dependency_satisfied(v2, satbility);
1403                                 }
1404 
1405                                 if (i == 1) {
1406                                         svc_satisfied = B_TRUE;
1407                                         break;
1408                                 }
1409                                 if (i == 0)
1410                                         svc_satisfiable = B_TRUE;
1411                         }
1412 
1413                         if (!svc_any_qualified)
1414                                 continue;
1415                         any_qualified = B_TRUE;
1416                         if (svc_satisfied) {
1417                                 i = 1;
1418                         } else if (svc_satisfiable) {
1419                                 i = 0;
1420                         } else {
1421                                 i = -1;
1422                         }
1423                         break;
1424                 }
1425 
1426                 case GVT_GROUP:
1427                 default:
1428 #ifndef NDEBUG
1429                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1430                             __LINE__, v->gv_type);
1431 #endif
1432                         abort();
1433                 }
1434 
1435                 if (i == 1)
1436                         continue;
1437 
1438                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1439                     "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1440                     v->gv_name, i == 0 ? "ed" : "able");
1441 
1442                 if (!satbility)
1443                         return (0);
1444                 if (i == -1)
1445                         return (-1);
1446                 any_unsatisfied = B_TRUE;
1447         }
1448 
1449         if (!any_qualified)
1450                 return (1);
1451 
1452         return (any_unsatisfied ? 0 : 1);
1453 }
1454 
1455 /*
1456  * An exclude_all dependency is unsatisfied if any non-service element is
1457  * satisfied or any service instance which is configured, enabled, and not in
1458  * maintenance is satisfied.  Usually when unsatisfied, it is also
1459  * unsatisfiable.
1460  */
1461 #define LOG_EXCLUDE(u, v)                                               \
1462         log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,                   \
1463             "exclude_all(%s): %s is satisfied.\n",                      \
1464             (u)->gv_name, (v)->gv_name)
1465 
1466 /* ARGSUSED */
1467 static int
1468 exclude_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1469 {
1470         graph_edge_t *edge, *e2;
1471         graph_vertex_t *v, *v2;
1472 
1473         for (edge = uu_list_first(groupv->gv_dependencies);
1474             edge != NULL;
1475             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1476                 v = edge->ge_vertex;
1477 
1478                 switch (v->gv_type) {
1479                 case GVT_INST:
1480                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1481                                 continue;
1482 
1483                         switch (v->gv_state) {
1484                         case RESTARTER_STATE_ONLINE:
1485                         case RESTARTER_STATE_DEGRADED:
1486                                 LOG_EXCLUDE(groupv, v);
1487                                 return (v->gv_flags & GV_ENABLED ? -1 : 0);
1488 
1489                         case RESTARTER_STATE_OFFLINE:
1490                         case RESTARTER_STATE_UNINIT:
1491                                 LOG_EXCLUDE(groupv, v);
1492                                 return (0);
1493 
1494                         case RESTARTER_STATE_DISABLED:
1495                         case RESTARTER_STATE_MAINT:
1496                                 continue;
1497 
1498                         default:
1499 #ifndef NDEBUG
1500                                 uu_warn("%s:%d: Unexpected vertex state %d.\n",
1501                                     __FILE__, __LINE__, v->gv_state);
1502 #endif
1503                                 abort();
1504                         }
1505                         /* NOTREACHED */
1506 
1507                 case GVT_SVC:
1508                         break;
1509 
1510                 case GVT_FILE:
1511                         if (!file_ready(v))
1512                                 continue;
1513                         LOG_EXCLUDE(groupv, v);
1514                         return (-1);
1515 
1516                 case GVT_GROUP:
1517                 default:
1518 #ifndef NDEBUG
1519                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1520                             __LINE__, v->gv_type);
1521 #endif
1522                         abort();
1523                 }
1524 
1525                 /* v represents a service */
1526                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1527                         continue;
1528 
1529                 for (e2 = uu_list_first(v->gv_dependencies);
1530                     e2 != NULL;
1531                     e2 = uu_list_next(v->gv_dependencies, e2)) {
1532                         v2 = e2->ge_vertex;
1533                         assert(v2->gv_type == GVT_INST);
1534 
1535                         if ((v2->gv_flags & GV_CONFIGURED) == 0)
1536                                 continue;
1537 
1538                         switch (v2->gv_state) {
1539                         case RESTARTER_STATE_ONLINE:
1540                         case RESTARTER_STATE_DEGRADED:
1541                                 LOG_EXCLUDE(groupv, v2);
1542                                 return (v2->gv_flags & GV_ENABLED ? -1 : 0);
1543 
1544                         case RESTARTER_STATE_OFFLINE:
1545                         case RESTARTER_STATE_UNINIT:
1546                                 LOG_EXCLUDE(groupv, v2);
1547                                 return (0);
1548 
1549                         case RESTARTER_STATE_DISABLED:
1550                         case RESTARTER_STATE_MAINT:
1551                                 continue;
1552 
1553                         default:
1554 #ifndef NDEBUG
1555                                 uu_warn("%s:%d: Unexpected vertex type %d.\n",
1556                                     __FILE__, __LINE__, v2->gv_type);
1557 #endif
1558                                 abort();
1559                         }
1560                 }
1561         }
1562 
1563         return (1);
1564 }
1565 
1566 /*
1567  * int instance_satisfied()
1568  *   Determine if all the dependencies are satisfied for the supplied instance
1569  *   vertex. Return 1 if they are, 0 if they aren't, and -1 if they won't be
1570  *   without administrator intervention.
1571  */
1572 static int
1573 instance_satisfied(graph_vertex_t *v, boolean_t satbility)
1574 {
1575         assert(v->gv_type == GVT_INST);
1576         assert(!inst_running(v));
1577 
1578         return (require_all_satisfied(v, satbility));
1579 }
1580 
1581 /*
1582  * Decide whether v can satisfy a dependency.  v can either be a child of
1583  * a group vertex, or of an instance vertex.
1584  */
1585 static int
1586 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1587 {
1588         switch (v->gv_type) {
1589         case GVT_INST:
1590                 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1591                         if (v->gv_flags & GV_DEATHROW) {
1592                                 /*
1593                                  * A dependency on an instance with GV_DEATHROW
1594                                  * flag is always considered as satisfied.
1595                                  */
1596                                 return (1);
1597                         }
1598                         return (-1);
1599                 }
1600 
1601                 /*
1602                  * Any vertex with the GV_TODISABLE flag set is guaranteed
1603                  * to have its dependencies unsatisfiable.  Any vertex with
1604                  * GV_TOOFFLINE may be satisfied after it transitions.
1605                  */
1606                 if (v->gv_flags & GV_TODISABLE)
1607                         return (-1);
1608                 if (v->gv_flags & GV_TOOFFLINE)
1609                         return (0);
1610 
1611                 switch (v->gv_state) {
1612                 case RESTARTER_STATE_ONLINE:
1613                 case RESTARTER_STATE_DEGRADED:
1614                         return (1);
1615 
1616                 case RESTARTER_STATE_OFFLINE:
1617                         if (!satbility)
1618                                 return (0);
1619                         return (instance_satisfied(v, satbility) != -1 ?
1620                             0 : -1);
1621 
1622                 case RESTARTER_STATE_DISABLED:
1623                 case RESTARTER_STATE_MAINT:
1624                         return (-1);
1625 
1626                 case RESTARTER_STATE_UNINIT:
1627                         return (0);
1628 
1629                 default:
1630 #ifndef NDEBUG
1631                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
1632                             __FILE__, __LINE__, v->gv_state);
1633 #endif
1634                         abort();
1635                         /* NOTREACHED */
1636                 }
1637 
1638         case GVT_SVC:
1639                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1640                         return (-1);
1641                 return (require_any_satisfied(v, satbility));
1642 
1643         case GVT_FILE:
1644                 /* i.e., we assume files will not be automatically generated */
1645                 return (file_ready(v) ? 1 : -1);
1646 
1647         case GVT_GROUP:
1648                 break;
1649 
1650         default:
1651 #ifndef NDEBUG
1652                 uu_warn("%s:%d: Unexpected node type %d.\n", __FILE__, __LINE__,
1653                     v->gv_type);
1654 #endif
1655                 abort();
1656                 /* NOTREACHED */
1657         }
1658 
1659         switch (v->gv_depgroup) {
1660         case DEPGRP_REQUIRE_ANY:
1661                 return (require_any_satisfied(v, satbility));
1662 
1663         case DEPGRP_REQUIRE_ALL:
1664                 return (require_all_satisfied(v, satbility));
1665 
1666         case DEPGRP_OPTIONAL_ALL:
1667                 return (optional_all_satisfied(v, satbility));
1668 
1669         case DEPGRP_EXCLUDE_ALL:
1670                 return (exclude_all_satisfied(v, satbility));
1671 
1672         default:
1673 #ifndef NDEBUG
1674                 uu_warn("%s:%d: Unknown dependency grouping %d.\n", __FILE__,
1675                     __LINE__, v->gv_depgroup);
1676 #endif
1677                 abort();
1678         }
1679 }
1680 
1681 void
1682 graph_start_if_satisfied(graph_vertex_t *v)
1683 {
1684         if (v->gv_state == RESTARTER_STATE_OFFLINE &&
1685             instance_satisfied(v, B_FALSE) == 1) {
1686                 if (v->gv_start_f == NULL)
1687                         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
1688                 else
1689                         v->gv_start_f(v);
1690         }
1691 }
1692 
1693 /*
1694  * propagate_satbility()
1695  *
1696  * This function is used when the given vertex changes state in such a way that
1697  * one of its dependents may become unsatisfiable.  This happens when an
1698  * instance transitions between offline -> online, or from !running ->
1699  * maintenance, as well as when an instance is removed from the graph.
1700  *
1701  * We have to walk all the dependents, since optional_all dependencies several
1702  * levels up could become (un)satisfied, instead of unsatisfiable.  For example,
1703  *
1704  *      +-----+  optional_all  +-----+  require_all  +-----+
1705  *      |  A  |--------------->|  B  |-------------->|  C  |
1706  *      +-----+                +-----+               +-----+
1707  *
1708  *                                              offline -> maintenance
1709  *
1710  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1711  * an optional dependency, what was previously an unsatisfiable situation is now
1712  * satisfied (B will never come online, even though its state hasn't changed).
1713  *
1714  * Note that it's not necessary to continue examining dependents after reaching
1715  * an optional_all dependency.  It's not possible for an optional_all dependency
1716  * to change satisfiability without also coming online, in which case we get a
1717  * start event and propagation continues naturally.  However, it does no harm to
1718  * continue propagating satisfiability (as it is a relatively rare event), and
1719  * keeps the walker code simple and generic.
1720  */
1721 /*ARGSUSED*/
1722 static int
1723 satbility_cb(graph_vertex_t *v, void *arg)
1724 {
1725         if (v->gv_flags & GV_TOOFFLINE)
1726                 return (UU_WALK_NEXT);
1727 
1728         if (v->gv_type == GVT_INST)
1729                 graph_start_if_satisfied(v);
1730 
1731         return (UU_WALK_NEXT);
1732 }
1733 
1734 static void
1735 propagate_satbility(graph_vertex_t *v)
1736 {
1737         graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1738 }
1739 
1740 static void propagate_stop(graph_vertex_t *, void *);
1741 
1742 /*
1743  * propagate_start()
1744  *
1745  * This function is used to propagate a start event to the dependents of the
1746  * given vertex.  Any dependents that are offline but have their dependencies
1747  * satisfied are started.  Any dependents that are online and have restart_on
1748  * set to "restart" or "refresh" are restarted because their dependencies have
1749  * just changed.  This only happens with optional_all dependencies.
1750  */
1751 static void
1752 propagate_start(graph_vertex_t *v, void *arg)
1753 {
1754         restarter_error_t err = (restarter_error_t)arg;
1755 
1756         if (v->gv_flags & GV_TOOFFLINE)
1757                 return;
1758 
1759         switch (v->gv_type) {
1760         case GVT_INST:
1761                 /* Restarter */
1762                 if (inst_running(v)) {
1763                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1764                                 vertex_send_event(v,
1765                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1766                         }
1767                 } else {
1768                         graph_start_if_satisfied(v);
1769                 }
1770                 break;
1771 
1772         case GVT_GROUP:
1773                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1774                         graph_walk_dependents(v, propagate_stop,
1775                             (void *)RERR_RESTART);
1776                         break;
1777                 }
1778                 err = v->gv_restart;
1779                 /* FALLTHROUGH */
1780 
1781         case GVT_SVC:
1782                 graph_walk_dependents(v, propagate_start, (void *)err);
1783                 break;
1784 
1785         case GVT_FILE:
1786 #ifndef NDEBUG
1787                 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1788                     __FILE__, __LINE__);
1789 #endif
1790                 abort();
1791                 /* NOTREACHED */
1792 
1793         default:
1794 #ifndef NDEBUG
1795                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1796                     v->gv_type);
1797 #endif
1798                 abort();
1799         }
1800 }
1801 
1802 /*
1803  * propagate_stop()
1804  *
1805  * This function is used to propagate a stop event to the dependencies of the
1806  * given vertex.  Any dependencies that are online (or in degraded state) with
1807  * the restart_on property set to "restart" or "refresh" will be restarted as
1808  * their dependencies have just changed.  This only happens for optional_all
1809  * dependencies.
1810  */
1811 static void
1812 propagate_stop(graph_vertex_t *v, void *arg)
1813 {
1814         restarter_error_t err = (restarter_error_t)arg;
1815 
1816         if (v->gv_flags & GV_TOOFFLINE)
1817                 return;
1818 
1819         switch (v->gv_type) {
1820         case GVT_INST:
1821                 /* Restarter */
1822                 if (err > RERR_NONE && inst_running(v)) {
1823                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1824                                 vertex_send_event(v,
1825                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1826                         } else {
1827                                 vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1828                         }
1829                 }
1830                 break;
1831 
1832         case GVT_SVC:
1833                 graph_walk_dependents(v, propagate_stop, arg);
1834                 break;
1835 
1836         case GVT_FILE:
1837 #ifndef NDEBUG
1838                 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1839                     __FILE__, __LINE__);
1840 #endif
1841                 abort();
1842                 /* NOTREACHED */
1843 
1844         case GVT_GROUP:
1845                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1846                         graph_walk_dependents(v, propagate_start, NULL);
1847                         break;
1848                 }
1849 
1850                 if (err == RERR_NONE || err > v->gv_restart)
1851                         break;
1852 
1853                 graph_walk_dependents(v, propagate_stop, arg);
1854                 break;
1855 
1856         default:
1857 #ifndef NDEBUG
1858                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1859                     v->gv_type);
1860 #endif
1861                 abort();
1862         }
1863 }
1864 
1865 void
1866 offline_vertex(graph_vertex_t *v)
1867 {
1868         scf_handle_t *h = libscf_handle_create_bound_loop();
1869         scf_instance_t *scf_inst = safe_scf_instance_create(h);
1870         scf_propertygroup_t *pg = safe_scf_pg_create(h);
1871         restarter_instance_state_t state, next_state;
1872         int r;
1873 
1874         assert(v->gv_type == GVT_INST);
1875 
1876         if (scf_inst == NULL)
1877                 bad_error("safe_scf_instance_create", scf_error());
1878         if (pg == NULL)
1879                 bad_error("safe_scf_pg_create", scf_error());
1880 
1881         /* if the vertex is already going offline, return */
1882 rep_retry:
1883         if (scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, scf_inst, NULL,
1884             NULL, SCF_DECODE_FMRI_EXACT) != 0) {
1885                 switch (scf_error()) {
1886                 case SCF_ERROR_CONNECTION_BROKEN:
1887                         libscf_handle_rebind(h);
1888                         goto rep_retry;
1889 
1890                 case SCF_ERROR_NOT_FOUND:
1891                         scf_pg_destroy(pg);
1892                         scf_instance_destroy(scf_inst);
1893                         (void) scf_handle_unbind(h);
1894                         scf_handle_destroy(h);
1895                         return;
1896                 }
1897                 uu_die("Can't decode FMRI %s: %s\n", v->gv_name,
1898                     scf_strerror(scf_error()));
1899         }
1900 
1901         r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
1902         if (r != 0) {
1903                 switch (scf_error()) {
1904                 case SCF_ERROR_CONNECTION_BROKEN:
1905                         libscf_handle_rebind(h);
1906                         goto rep_retry;
1907 
1908                 case SCF_ERROR_NOT_SET:
1909                 case SCF_ERROR_NOT_FOUND:
1910                         scf_pg_destroy(pg);
1911                         scf_instance_destroy(scf_inst);
1912                         (void) scf_handle_unbind(h);
1913                         scf_handle_destroy(h);
1914                         return;
1915 
1916                 default:
1917                         bad_error("scf_instance_get_pg", scf_error());
1918                 }
1919         } else {
1920                 r = libscf_read_states(pg, &state, &next_state);
1921                 if (r == 0 && (next_state == RESTARTER_STATE_OFFLINE ||
1922                     next_state == RESTARTER_STATE_DISABLED)) {
1923                         log_framework(LOG_DEBUG,
1924                             "%s: instance is already going down.\n",
1925                             v->gv_name);
1926                         scf_pg_destroy(pg);
1927                         scf_instance_destroy(scf_inst);
1928                         (void) scf_handle_unbind(h);
1929                         scf_handle_destroy(h);
1930                         return;
1931                 }
1932         }
1933 
1934         scf_pg_destroy(pg);
1935         scf_instance_destroy(scf_inst);
1936         (void) scf_handle_unbind(h);
1937         scf_handle_destroy(h);
1938 
1939         vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP_RESET);
1940 }
1941 
1942 /*
1943  * void graph_enable_by_vertex()
1944  *   If admin is non-zero, this is an administrative request for change
1945  *   of the enabled property.  Thus, send the ADMIN_DISABLE rather than
1946  *   a plain DISABLE restarter event.
1947  */
1948 void
1949 graph_enable_by_vertex(graph_vertex_t *vertex, int enable, int admin)
1950 {
1951         graph_vertex_t *v;
1952         int r;
1953 
1954         assert(MUTEX_HELD(&dgraph_lock));
1955         assert((vertex->gv_flags & GV_CONFIGURED));
1956 
1957         vertex->gv_flags = (vertex->gv_flags & ~GV_ENABLED) |
1958             (enable ? GV_ENABLED : 0);
1959 
1960         if (enable) {
1961                 if (vertex->gv_state != RESTARTER_STATE_OFFLINE &&
1962                     vertex->gv_state != RESTARTER_STATE_DEGRADED &&
1963                     vertex->gv_state != RESTARTER_STATE_ONLINE) {
1964                         /*
1965                          * In case the vertex was notified to go down,
1966                          * but now can return online, clear the _TOOFFLINE
1967                          * and _TODISABLE flags.
1968                          */
1969                         vertex->gv_flags &= ~GV_TOOFFLINE;
1970                         vertex->gv_flags &= ~GV_TODISABLE;
1971 
1972                         vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ENABLE);
1973                 }
1974 
1975                 /*
1976                  * Wait for state update from restarter before sending _START or
1977                  * _STOP.
1978                  */
1979 
1980                 return;
1981         }
1982 
1983         if (vertex->gv_state == RESTARTER_STATE_DISABLED)
1984                 return;
1985 
1986         if (!admin) {
1987                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_DISABLE);
1988 
1989                 /*
1990                  * Wait for state update from restarter before sending _START or
1991                  * _STOP.
1992                  */
1993 
1994                 return;
1995         }
1996 
1997         /*
1998          * If it is a DISABLE event requested by the administrator then we are
1999          * offlining the dependents first.
2000          */
2001 
2002         /*
2003          * Set GV_TOOFFLINE for the services we are offlining. We cannot
2004          * clear the GV_TOOFFLINE bits from all the services because
2005          * other DISABLE events might be handled at the same time.
2006          */
2007         vertex->gv_flags |= GV_TOOFFLINE;
2008 
2009         /* remember which vertex to disable... */
2010         vertex->gv_flags |= GV_TODISABLE;
2011 
2012         log_framework(LOG_DEBUG, "Marking in-subtree vertices before "
2013             "disabling %s.\n", vertex->gv_name);
2014 
2015         /* set GV_TOOFFLINE for its dependents */
2016         r = uu_list_walk(vertex->gv_dependents, (uu_walk_fn_t *)mark_subtree,
2017             NULL, 0);
2018         assert(r == 0);
2019 
2020         /* disable the instance now if there is nothing else to offline */
2021         if (insubtree_dependents_down(vertex) == B_TRUE) {
2022                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
2023                 return;
2024         }
2025 
2026         /*
2027          * This loop is similar to the one used for the graph reversal shutdown
2028          * and could be improved in term of performance for the subtree reversal
2029          * disable case.
2030          */
2031         for (v = uu_list_first(dgraph); v != NULL;
2032             v = uu_list_next(dgraph, v)) {
2033                 /* skip the vertex we are disabling for now */
2034                 if (v == vertex)
2035                         continue;
2036 
2037                 if (v->gv_type != GVT_INST ||
2038                     (v->gv_flags & GV_CONFIGURED) == 0 ||
2039                     (v->gv_flags & GV_ENABLED) == 0 ||
2040                     (v->gv_flags & GV_TOOFFLINE) == 0)
2041                         continue;
2042 
2043                 if ((v->gv_state != RESTARTER_STATE_ONLINE) &&
2044                     (v->gv_state != RESTARTER_STATE_DEGRADED)) {
2045                         /* continue if there is nothing to offline */
2046                         continue;
2047                 }
2048 
2049                 /*
2050                  * Instances which are up need to come down before we're
2051                  * done, but we can only offline the leaves here. An
2052                  * instance is a leaf when all its dependents are down.
2053                  */
2054                 if (insubtree_dependents_down(v) == B_TRUE) {
2055                         log_framework(LOG_DEBUG, "Offlining in-subtree "
2056                             "instance %s for %s.\n",
2057                             v->gv_name, vertex->gv_name);
2058                         offline_vertex(v);
2059                 }
2060         }
2061 }
2062 
2063 static int configure_vertex(graph_vertex_t *, scf_instance_t *);
2064 
2065 /*
2066  * Set the restarter for v to fmri_arg.  That is, make sure a vertex for
2067  * fmri_arg exists, make v depend on it, and send _ADD_INSTANCE for v.  If
2068  * v is already configured and fmri_arg indicates the current restarter, do
2069  * nothing.  If v is configured and fmri_arg is a new restarter, delete v's
2070  * dependency on the restarter, send _REMOVE_INSTANCE for v, and set the new
2071  * restarter.  Returns 0 on success, EINVAL if the FMRI is invalid,
2072  * ECONNABORTED if the repository connection is broken, and ELOOP
2073  * if the dependency would create a cycle.  In the last case, *pathp will
2074  * point to a -1-terminated array of ids which compose the path from v to
2075  * restarter_fmri.
2076  */
2077 int
2078 graph_change_restarter(graph_vertex_t *v, const char *fmri_arg, scf_handle_t *h,
2079     int **pathp)
2080 {
2081         char *restarter_fmri = NULL;
2082         graph_vertex_t *rv;
2083         int err;
2084         int id;
2085 
2086         assert(MUTEX_HELD(&dgraph_lock));
2087 
2088         if (fmri_arg[0] != '\0') {
2089                 err = fmri_canonify(fmri_arg, &restarter_fmri, B_TRUE);
2090                 if (err != 0) {
2091                         assert(err == EINVAL);
2092                         return (err);
2093                 }
2094         }
2095 
2096         if (restarter_fmri == NULL ||
2097             strcmp(restarter_fmri, SCF_SERVICE_STARTD) == 0) {
2098                 if (v->gv_flags & GV_CONFIGURED) {
2099                         if (v->gv_restarter_id == -1) {
2100                                 if (restarter_fmri != NULL)
2101                                         startd_free(restarter_fmri,
2102                                             max_scf_fmri_size);
2103                                 return (0);
2104                         }
2105 
2106                         graph_unset_restarter(v);
2107                 }
2108 
2109                 /* Master restarter, nothing to do. */
2110                 v->gv_restarter_id = -1;
2111                 v->gv_restarter_channel = NULL;
2112                 vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2113                 return (0);
2114         }
2115 
2116         if (v->gv_flags & GV_CONFIGURED) {
2117                 id = dict_lookup_byname(restarter_fmri);
2118                 if (id != -1 && v->gv_restarter_id == id) {
2119                         startd_free(restarter_fmri, max_scf_fmri_size);
2120                         return (0);
2121                 }
2122 
2123                 graph_unset_restarter(v);
2124         }
2125 
2126         err = graph_insert_vertex_unconfigured(restarter_fmri, GVT_INST, 0,
2127             RERR_NONE, &rv);
2128         startd_free(restarter_fmri, max_scf_fmri_size);
2129         assert(err == 0 || err == EEXIST);
2130 
2131         if (rv->gv_delegate_initialized == 0) {
2132                 if ((rv->gv_delegate_channel = restarter_protocol_init_delegate(
2133                     rv->gv_name)) == NULL)
2134                         return (EINVAL);
2135                 rv->gv_delegate_initialized = 1;
2136         }
2137         v->gv_restarter_id = rv->gv_id;
2138         v->gv_restarter_channel = rv->gv_delegate_channel;
2139 
2140         err = graph_insert_dependency(v, rv, pathp);
2141         if (err != 0) {
2142                 assert(err == ELOOP);
2143                 return (ELOOP);
2144         }
2145 
2146         vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2147 
2148         if (!(rv->gv_flags & GV_CONFIGURED)) {
2149                 scf_instance_t *inst;
2150 
2151                 err = libscf_fmri_get_instance(h, rv->gv_name, &inst);
2152                 switch (err) {
2153                 case 0:
2154                         err = configure_vertex(rv, inst);
2155                         scf_instance_destroy(inst);
2156                         switch (err) {
2157                         case 0:
2158                         case ECANCELED:
2159                                 break;
2160 
2161                         case ECONNABORTED:
2162                                 return (ECONNABORTED);
2163 
2164                         default:
2165                                 bad_error("configure_vertex", err);
2166                         }
2167                         break;
2168 
2169                 case ECONNABORTED:
2170                         return (ECONNABORTED);
2171 
2172                 case ENOENT:
2173                         break;
2174 
2175                 case ENOTSUP:
2176                         /*
2177                          * The fmri doesn't specify an instance - translate
2178                          * to EINVAL.
2179                          */
2180                         return (EINVAL);
2181 
2182                 case EINVAL:
2183                 default:
2184                         bad_error("libscf_fmri_get_instance", err);
2185                 }
2186         }
2187 
2188         return (0);
2189 }
2190 
2191 
2192 /*
2193  * Add all of the instances of the service named by fmri to the graph.
2194  * Returns
2195  *   0 - success
2196  *   ENOENT - service indicated by fmri does not exist
2197  *
2198  * In both cases *reboundp will be B_TRUE if the handle was rebound, or B_FALSE
2199  * otherwise.
2200  */
2201 static int
2202 add_service(const char *fmri, scf_handle_t *h, boolean_t *reboundp)
2203 {
2204         scf_service_t *svc;
2205         scf_instance_t *inst;
2206         scf_iter_t *iter;
2207         char *inst_fmri;
2208         int ret, r;
2209 
2210         *reboundp = B_FALSE;
2211 
2212         svc = safe_scf_service_create(h);
2213         inst = safe_scf_instance_create(h);
2214         iter = safe_scf_iter_create(h);
2215         inst_fmri = startd_alloc(max_scf_fmri_size);
2216 
2217 rebound:
2218         if (scf_handle_decode_fmri(h, fmri, NULL, svc, NULL, NULL, NULL,
2219             SCF_DECODE_FMRI_EXACT) != 0) {
2220                 switch (scf_error()) {
2221                 case SCF_ERROR_CONNECTION_BROKEN:
2222                 default:
2223                         libscf_handle_rebind(h);
2224                         *reboundp = B_TRUE;
2225                         goto rebound;
2226 
2227                 case SCF_ERROR_NOT_FOUND:
2228                         ret = ENOENT;
2229                         goto out;
2230 
2231                 case SCF_ERROR_INVALID_ARGUMENT:
2232                 case SCF_ERROR_CONSTRAINT_VIOLATED:
2233                 case SCF_ERROR_NOT_BOUND:
2234                 case SCF_ERROR_HANDLE_MISMATCH:
2235                         bad_error("scf_handle_decode_fmri", scf_error());
2236                 }
2237         }
2238 
2239         if (scf_iter_service_instances(iter, svc) != 0) {
2240                 switch (scf_error()) {
2241                 case SCF_ERROR_CONNECTION_BROKEN:
2242                 default:
2243                         libscf_handle_rebind(h);
2244                         *reboundp = B_TRUE;
2245                         goto rebound;
2246 
2247                 case SCF_ERROR_DELETED:
2248                         ret = ENOENT;
2249                         goto out;
2250 
2251                 case SCF_ERROR_HANDLE_MISMATCH:
2252                 case SCF_ERROR_NOT_BOUND:
2253                 case SCF_ERROR_NOT_SET:
2254                         bad_error("scf_iter_service_instances", scf_error());
2255                 }
2256         }
2257 
2258         for (;;) {
2259                 r = scf_iter_next_instance(iter, inst);
2260                 if (r == 0)
2261                         break;
2262                 if (r != 1) {
2263                         switch (scf_error()) {
2264                         case SCF_ERROR_CONNECTION_BROKEN:
2265                         default:
2266                                 libscf_handle_rebind(h);
2267                                 *reboundp = B_TRUE;
2268                                 goto rebound;
2269 
2270                         case SCF_ERROR_DELETED:
2271                                 ret = ENOENT;
2272                                 goto out;
2273 
2274                         case SCF_ERROR_HANDLE_MISMATCH:
2275                         case SCF_ERROR_NOT_BOUND:
2276                         case SCF_ERROR_NOT_SET:
2277                         case SCF_ERROR_INVALID_ARGUMENT:
2278                                 bad_error("scf_iter_next_instance",
2279                                     scf_error());
2280                         }
2281                 }
2282 
2283                 if (scf_instance_to_fmri(inst, inst_fmri, max_scf_fmri_size) <
2284                     0) {
2285                         switch (scf_error()) {
2286                         case SCF_ERROR_CONNECTION_BROKEN:
2287                                 libscf_handle_rebind(h);
2288                                 *reboundp = B_TRUE;
2289                                 goto rebound;
2290 
2291                         case SCF_ERROR_DELETED:
2292                                 continue;
2293 
2294                         case SCF_ERROR_NOT_BOUND:
2295                         case SCF_ERROR_NOT_SET:
2296                                 bad_error("scf_instance_to_fmri", scf_error());
2297                         }
2298                 }
2299 
2300                 r = dgraph_add_instance(inst_fmri, inst, B_FALSE);
2301                 switch (r) {
2302                 case 0:
2303                 case ECANCELED:
2304                         break;
2305 
2306                 case EEXIST:
2307                         continue;
2308 
2309                 case ECONNABORTED:
2310                         libscf_handle_rebind(h);
2311                         *reboundp = B_TRUE;
2312                         goto rebound;
2313 
2314                 case EINVAL:
2315                 default:
2316                         bad_error("dgraph_add_instance", r);
2317                 }
2318         }
2319 
2320         ret = 0;
2321 
2322 out:
2323         startd_free(inst_fmri, max_scf_fmri_size);
2324         scf_iter_destroy(iter);
2325         scf_instance_destroy(inst);
2326         scf_service_destroy(svc);
2327         return (ret);
2328 }
2329 
2330 struct depfmri_info {
2331         graph_vertex_t  *v;             /* GVT_GROUP vertex */
2332         gv_type_t       type;           /* type of dependency */
2333         const char      *inst_fmri;     /* FMRI of parental GVT_INST vert. */
2334         const char      *pg_name;       /* Name of dependency pg */
2335         scf_handle_t    *h;
2336         int             err;            /* return error code */
2337         int             **pathp;        /* return circular dependency path */
2338 };
2339 
2340 /*
2341  * Find or create a vertex for fmri and make info->v depend on it.
2342  * Returns
2343  *   0 - success
2344  *   nonzero - failure
2345  *
2346  * On failure, sets info->err to
2347  *   EINVAL - fmri is invalid
2348  *            fmri does not match info->type
2349  *   ELOOP - Adding the dependency creates a circular dependency.  *info->pathp
2350  *           will point to an array of the ids of the members of the cycle.
2351  *   ECONNABORTED - repository connection was broken
2352  *   ECONNRESET - succeeded, but repository connection was reset
2353  */
2354 static int
2355 process_dependency_fmri(const char *fmri, struct depfmri_info *info)
2356 {
2357         int err;
2358         graph_vertex_t *depgroup_v, *v;
2359         char *fmri_copy, *cfmri;
2360         size_t fmri_copy_sz;
2361         const char *scope, *service, *instance, *pg;
2362         scf_instance_t *inst;
2363         boolean_t rebound;
2364 
2365         assert(MUTEX_HELD(&dgraph_lock));
2366 
2367         /* Get or create vertex for FMRI */
2368         depgroup_v = info->v;
2369 
2370         if (strncmp(fmri, "file:", sizeof ("file:") - 1) == 0) {
2371                 if (info->type != GVT_FILE) {
2372                         log_framework(LOG_NOTICE,
2373                             "FMRI \"%s\" is not allowed for the \"%s\" "
2374                             "dependency's type of instance %s.\n", fmri,
2375                             info->pg_name, info->inst_fmri);
2376                         return (info->err = EINVAL);
2377                 }
2378 
2379                 err = graph_insert_vertex_unconfigured(fmri, info->type, 0,
2380                     RERR_NONE, &v);
2381                 switch (err) {
2382                 case 0:
2383                         break;
2384 
2385                 case EEXIST:
2386                         assert(v->gv_type == GVT_FILE);
2387                         break;
2388 
2389                 case EINVAL:            /* prevented above */
2390                 default:
2391                         bad_error("graph_insert_vertex_unconfigured", err);
2392                 }
2393         } else {
2394                 if (info->type != GVT_INST) {
2395                         log_framework(LOG_NOTICE,
2396                             "FMRI \"%s\" is not allowed for the \"%s\" "
2397                             "dependency's type of instance %s.\n", fmri,
2398                             info->pg_name, info->inst_fmri);
2399                         return (info->err = EINVAL);
2400                 }
2401 
2402                 /*
2403                  * We must canonify fmri & add a vertex for it.
2404                  */
2405                 fmri_copy_sz = strlen(fmri) + 1;
2406                 fmri_copy = startd_alloc(fmri_copy_sz);
2407                 (void) strcpy(fmri_copy, fmri);
2408 
2409                 /* Determine if the FMRI is a property group or instance */
2410                 if (scf_parse_svc_fmri(fmri_copy, &scope, &service,
2411                     &instance, &pg, NULL) != 0) {
2412                         startd_free(fmri_copy, fmri_copy_sz);
2413                         log_framework(LOG_NOTICE,
2414                             "Dependency \"%s\" of %s has invalid FMRI "
2415                             "\"%s\".\n", info->pg_name, info->inst_fmri,
2416                             fmri);
2417                         return (info->err = EINVAL);
2418                 }
2419 
2420                 if (service == NULL || pg != NULL) {
2421                         startd_free(fmri_copy, fmri_copy_sz);
2422                         log_framework(LOG_NOTICE,
2423                             "Dependency \"%s\" of %s does not designate a "
2424                             "service or instance.\n", info->pg_name,
2425                             info->inst_fmri);
2426                         return (info->err = EINVAL);
2427                 }
2428 
2429                 if (scope == NULL || strcmp(scope, SCF_SCOPE_LOCAL) == 0) {
2430                         cfmri = uu_msprintf("svc:/%s%s%s",
2431                             service, instance ? ":" : "", instance ? instance :
2432                             "");
2433                 } else {
2434                         cfmri = uu_msprintf("svc://%s/%s%s%s",
2435                             scope, service, instance ? ":" : "", instance ?
2436                             instance : "");
2437                 }
2438 
2439                 startd_free(fmri_copy, fmri_copy_sz);
2440 
2441                 err = graph_insert_vertex_unconfigured(cfmri, instance ?
2442                     GVT_INST : GVT_SVC, instance ? 0 : DEPGRP_REQUIRE_ANY,
2443                     RERR_NONE, &v);
2444                 uu_free(cfmri);
2445                 switch (err) {
2446                 case 0:
2447                         break;
2448 
2449                 case EEXIST:
2450                         /* Verify v. */
2451                         if (instance != NULL)
2452                                 assert(v->gv_type == GVT_INST);
2453                         else
2454                                 assert(v->gv_type == GVT_SVC);
2455                         break;
2456 
2457                 default:
2458                         bad_error("graph_insert_vertex_unconfigured", err);
2459                 }
2460         }
2461 
2462         /* Add dependency from depgroup_v to new vertex */
2463         info->err = graph_insert_dependency(depgroup_v, v, info->pathp);
2464         switch (info->err) {
2465         case 0:
2466                 break;
2467 
2468         case ELOOP:
2469                 return (ELOOP);
2470 
2471         default:
2472                 bad_error("graph_insert_dependency", info->err);
2473         }
2474 
2475         /* This must be after we insert the dependency, to avoid looping. */
2476         switch (v->gv_type) {
2477         case GVT_INST:
2478                 if ((v->gv_flags & GV_CONFIGURED) != 0)
2479                         break;
2480 
2481                 inst = safe_scf_instance_create(info->h);
2482 
2483                 rebound = B_FALSE;
2484 
2485 rebound:
2486                 err = libscf_lookup_instance(v->gv_name, inst);
2487                 switch (err) {
2488                 case 0:
2489                         err = configure_vertex(v, inst);
2490                         switch (err) {
2491                         case 0:
2492                         case ECANCELED:
2493                                 break;
2494 
2495                         case ECONNABORTED:
2496                                 libscf_handle_rebind(info->h);
2497                                 rebound = B_TRUE;
2498                                 goto rebound;
2499 
2500                         default:
2501                                 bad_error("configure_vertex", err);
2502                         }
2503                         break;
2504 
2505                 case ENOENT:
2506                         break;
2507 
2508                 case ECONNABORTED:
2509                         libscf_handle_rebind(info->h);
2510                         rebound = B_TRUE;
2511                         goto rebound;
2512 
2513                 case EINVAL:
2514                 case ENOTSUP:
2515                 default:
2516                         bad_error("libscf_fmri_get_instance", err);
2517                 }
2518 
2519                 scf_instance_destroy(inst);
2520 
2521                 if (rebound)
2522                         return (info->err = ECONNRESET);
2523                 break;
2524 
2525         case GVT_SVC:
2526                 (void) add_service(v->gv_name, info->h, &rebound);
2527                 if (rebound)
2528                         return (info->err = ECONNRESET);
2529         }
2530 
2531         return (0);
2532 }
2533 
2534 struct deppg_info {
2535         graph_vertex_t  *v;             /* GVT_INST vertex */
2536         int             err;            /* return error */
2537         int             **pathp;        /* return circular dependency path */
2538 };
2539 
2540 /*
2541  * Make info->v depend on a new GVT_GROUP node for this property group,
2542  * and then call process_dependency_fmri() for the values of the entity
2543  * property.  Return 0 on success, or if something goes wrong return nonzero
2544  * and set info->err to ECONNABORTED, EINVAL, or the error code returned by
2545  * process_dependency_fmri().
2546  */
2547 static int
2548 process_dependency_pg(scf_propertygroup_t *pg, struct deppg_info *info)
2549 {
2550         scf_handle_t *h;
2551         depgroup_type_t deptype;
2552         restarter_error_t rerr;
2553         struct depfmri_info linfo;
2554         char *fmri, *pg_name;
2555         size_t fmri_sz;
2556         graph_vertex_t *depgrp;
2557         scf_property_t *prop;
2558         int err;
2559         int empty;
2560         scf_error_t scferr;
2561         ssize_t len;
2562 
2563         assert(MUTEX_HELD(&dgraph_lock));
2564 
2565         h = scf_pg_handle(pg);
2566 
2567         pg_name = startd_alloc(max_scf_name_size);
2568 
2569         len = scf_pg_get_name(pg, pg_name, max_scf_name_size);
2570         if (len < 0) {
2571                 startd_free(pg_name, max_scf_name_size);
2572                 switch (scf_error()) {
2573                 case SCF_ERROR_CONNECTION_BROKEN:
2574                 default:
2575                         return (info->err = ECONNABORTED);
2576 
2577                 case SCF_ERROR_DELETED:
2578                         return (info->err = 0);
2579 
2580                 case SCF_ERROR_NOT_SET:
2581                         bad_error("scf_pg_get_name", scf_error());
2582                 }
2583         }
2584 
2585         /*
2586          * Skip over empty dependency groups.  Since dependency property
2587          * groups are updated atomically, they are either empty or
2588          * fully populated.
2589          */
2590         empty = depgroup_empty(h, pg);
2591         if (empty < 0) {
2592                 log_error(LOG_INFO,
2593                     "Error reading dependency group \"%s\" of %s: %s\n",
2594                     pg_name, info->v->gv_name, scf_strerror(scf_error()));
2595                 startd_free(pg_name, max_scf_name_size);
2596                 return (info->err = EINVAL);
2597 
2598         } else if (empty == 1) {
2599                 log_framework(LOG_DEBUG,
2600                     "Ignoring empty dependency group \"%s\" of %s\n",
2601                     pg_name, info->v->gv_name);
2602                 startd_free(pg_name, max_scf_name_size);
2603                 return (info->err = 0);
2604         }
2605 
2606         fmri_sz = strlen(info->v->gv_name) + 1 + len + 1;
2607         fmri = startd_alloc(fmri_sz);
2608 
2609         (void) snprintf(fmri, fmri_sz, "%s>%s", info->v->gv_name,
2610             pg_name);
2611 
2612         /* Validate the pg before modifying the graph */
2613         deptype = depgroup_read_grouping(h, pg);
2614         if (deptype == DEPGRP_UNSUPPORTED) {
2615                 log_error(LOG_INFO,
2616                     "Dependency \"%s\" of %s has an unknown grouping value.\n",
2617                     pg_name, info->v->gv_name);
2618                 startd_free(fmri, fmri_sz);
2619                 startd_free(pg_name, max_scf_name_size);
2620                 return (info->err = EINVAL);
2621         }
2622 
2623         rerr = depgroup_read_restart(h, pg);
2624         if (rerr == RERR_UNSUPPORTED) {
2625                 log_error(LOG_INFO,
2626                     "Dependency \"%s\" of %s has an unknown restart_on value."
2627                     "\n", pg_name, info->v->gv_name);
2628                 startd_free(fmri, fmri_sz);
2629                 startd_free(pg_name, max_scf_name_size);
2630                 return (info->err = EINVAL);
2631         }
2632 
2633         prop = safe_scf_property_create(h);
2634 
2635         if (scf_pg_get_property(pg, SCF_PROPERTY_ENTITIES, prop) != 0) {
2636                 scferr = scf_error();
2637                 scf_property_destroy(prop);
2638                 if (scferr == SCF_ERROR_DELETED) {
2639                         startd_free(fmri, fmri_sz);
2640                         startd_free(pg_name, max_scf_name_size);
2641                         return (info->err = 0);
2642                 } else if (scferr != SCF_ERROR_NOT_FOUND) {
2643                         startd_free(fmri, fmri_sz);
2644                         startd_free(pg_name, max_scf_name_size);
2645                         return (info->err = ECONNABORTED);
2646                 }
2647 
2648                 log_error(LOG_INFO,
2649                     "Dependency \"%s\" of %s is missing a \"%s\" property.\n",
2650                     pg_name, info->v->gv_name, SCF_PROPERTY_ENTITIES);
2651 
2652                 startd_free(fmri, fmri_sz);
2653                 startd_free(pg_name, max_scf_name_size);
2654 
2655                 return (info->err = EINVAL);
2656         }
2657 
2658         /* Create depgroup vertex for pg */
2659         err = graph_insert_vertex_unconfigured(fmri, GVT_GROUP, deptype,
2660             rerr, &depgrp);
2661         assert(err == 0);
2662         startd_free(fmri, fmri_sz);
2663 
2664         /* Add dependency from inst vertex to new vertex */
2665         err = graph_insert_dependency(info->v, depgrp, info->pathp);
2666         /* ELOOP can't happen because this should be a new vertex */
2667         assert(err == 0);
2668 
2669         linfo.v = depgrp;
2670         linfo.type = depgroup_read_scheme(h, pg);
2671         linfo.inst_fmri = info->v->gv_name;
2672         linfo.pg_name = pg_name;
2673         linfo.h = h;
2674         linfo.err = 0;
2675         linfo.pathp = info->pathp;
2676         err = walk_property_astrings(prop, (callback_t)process_dependency_fmri,
2677             &linfo);
2678 
2679         scf_property_destroy(prop);
2680         startd_free(pg_name, max_scf_name_size);
2681 
2682         switch (err) {
2683         case 0:
2684         case EINTR:
2685                 return (info->err = linfo.err);
2686 
2687         case ECONNABORTED:
2688         case EINVAL:
2689                 return (info->err = err);
2690 
2691         case ECANCELED:
2692                 return (info->err = 0);
2693 
2694         case ECONNRESET:
2695                 return (info->err = ECONNABORTED);
2696 
2697         default:
2698                 bad_error("walk_property_astrings", err);
2699                 /* NOTREACHED */
2700         }
2701 }
2702 
2703 /*
2704  * Build the dependency info for v from the repository.  Returns 0 on success,
2705  * ECONNABORTED on repository disconnection, EINVAL if the repository
2706  * configuration is invalid, and ELOOP if a dependency would cause a cycle.
2707  * In the last case, *pathp will point to a -1-terminated array of ids which
2708  * constitute the rest of the dependency cycle.
2709  */
2710 static int
2711 set_dependencies(graph_vertex_t *v, scf_instance_t *inst, int **pathp)
2712 {
2713         struct deppg_info info;
2714         int err;
2715         uint_t old_configured;
2716 
2717         assert(MUTEX_HELD(&dgraph_lock));
2718 
2719         /*
2720          * Mark the vertex as configured during dependency insertion to avoid
2721          * dependency cycles (which can appear in the graph if one of the
2722          * vertices is an exclusion-group).
2723          */
2724         old_configured = v->gv_flags & GV_CONFIGURED;
2725         v->gv_flags |= GV_CONFIGURED;
2726 
2727         info.err = 0;
2728         info.v = v;
2729         info.pathp = pathp;
2730 
2731         err = walk_dependency_pgs(inst, (callback_t)process_dependency_pg,
2732             &info);
2733 
2734         if (!old_configured)
2735                 v->gv_flags &= ~GV_CONFIGURED;
2736 
2737         switch (err) {
2738         case 0:
2739         case EINTR:
2740                 return (info.err);
2741 
2742         case ECONNABORTED:
2743                 return (ECONNABORTED);
2744 
2745         case ECANCELED:
2746                 /* Should get delete event, so return 0. */
2747                 return (0);
2748 
2749         default:
2750                 bad_error("walk_dependency_pgs", err);
2751                 /* NOTREACHED */
2752         }
2753 }
2754 
2755 
2756 static void
2757 handle_cycle(const char *fmri, int *path)
2758 {
2759         const char *cp;
2760         size_t sz;
2761 
2762         assert(MUTEX_HELD(&dgraph_lock));
2763 
2764         path_to_str(path, (char **)&cp, &sz);
2765 
2766         log_error(LOG_ERR, "Transitioning %s to maintenance "
2767             "because it completes a dependency cycle (see svcs -xv for "
2768             "details):\n%s", fmri ? fmri : "?", cp);
2769 
2770         startd_free((void *)cp, sz);
2771 }
2772 
2773 /*
2774  * Increment the vertex's reference count to prevent the vertex removal
2775  * from the dgraph.
2776  */
2777 static void
2778 vertex_ref(graph_vertex_t *v)
2779 {
2780         assert(MUTEX_HELD(&dgraph_lock));
2781 
2782         v->gv_refs++;
2783 }
2784 
2785 /*
2786  * Decrement the vertex's reference count and remove the vertex from
2787  * the dgraph when possible.
2788  *
2789  * Return VERTEX_REMOVED when the vertex has been removed otherwise
2790  * return VERTEX_INUSE.
2791  */
2792 static int
2793 vertex_unref(graph_vertex_t *v)
2794 {
2795         assert(MUTEX_HELD(&dgraph_lock));
2796         assert(v->gv_refs > 0);
2797 
2798         v->gv_refs--;
2799 
2800         return (free_if_unrefed(v));
2801 }
2802 
2803 /*
2804  * When run on the dependencies of a vertex, populates list with
2805  * graph_edge_t's which point to the service vertices or the instance
2806  * vertices (no GVT_GROUP nodes) on which the vertex depends.
2807  *
2808  * Increment the vertex's reference count once the vertex is inserted
2809  * in the list. The vertex won't be able to be deleted from the dgraph
2810  * while it is referenced.
2811  */
2812 static int
2813 append_svcs_or_insts(graph_edge_t *e, uu_list_t *list)
2814 {
2815         graph_vertex_t *v = e->ge_vertex;
2816         graph_edge_t *new;
2817         int r;
2818 
2819         switch (v->gv_type) {
2820         case GVT_INST:
2821         case GVT_SVC:
2822                 break;
2823 
2824         case GVT_GROUP:
2825                 r = uu_list_walk(v->gv_dependencies,
2826                     (uu_walk_fn_t *)append_svcs_or_insts, list, 0);
2827                 assert(r == 0);
2828                 return (UU_WALK_NEXT);
2829 
2830         case GVT_FILE:
2831                 return (UU_WALK_NEXT);
2832 
2833         default:
2834 #ifndef NDEBUG
2835                 uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
2836                     __LINE__, v->gv_type);
2837 #endif
2838                 abort();
2839         }
2840 
2841         new = startd_alloc(sizeof (*new));
2842         new->ge_vertex = v;
2843         uu_list_node_init(new, &new->ge_link, graph_edge_pool);
2844         r = uu_list_insert_before(list, NULL, new);
2845         assert(r == 0);
2846 
2847         /*
2848          * Because we are inserting the vertex in a list, we don't want
2849          * the vertex to be freed while the list is in use. In order to
2850          * achieve that, increment the vertex's reference count.
2851          */
2852         vertex_ref(v);
2853 
2854         return (UU_WALK_NEXT);
2855 }
2856 
2857 static boolean_t
2858 should_be_in_subgraph(graph_vertex_t *v)
2859 {
2860         graph_edge_t *e;
2861 
2862         if (v == milestone)
2863                 return (B_TRUE);
2864 
2865         /*
2866          * v is in the subgraph if any of its dependents are in the subgraph.
2867          * Except for EXCLUDE_ALL dependents.  And OPTIONAL dependents only
2868          * count if we're enabled.
2869          */
2870         for (e = uu_list_first(v->gv_dependents);
2871             e != NULL;
2872             e = uu_list_next(v->gv_dependents, e)) {
2873                 graph_vertex_t *dv = e->ge_vertex;
2874 
2875                 if (!(dv->gv_flags & GV_INSUBGRAPH))
2876                         continue;
2877 
2878                 /*
2879                  * Don't include instances that are optional and disabled.
2880                  */
2881                 if (v->gv_type == GVT_INST && dv->gv_type == GVT_SVC) {
2882 
2883                         int in = 0;
2884                         graph_edge_t *ee;
2885 
2886                         for (ee = uu_list_first(dv->gv_dependents);
2887                             ee != NULL;
2888                             ee = uu_list_next(dv->gv_dependents, ee)) {
2889 
2890                                 graph_vertex_t *ddv = e->ge_vertex;
2891 
2892                                 if (ddv->gv_type == GVT_GROUP &&
2893                                     ddv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2894                                         continue;
2895 
2896                                 if (ddv->gv_type == GVT_GROUP &&
2897                                     ddv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2898                                     !(v->gv_flags & GV_ENBLD_NOOVR))
2899                                         continue;
2900 
2901                                 in = 1;
2902                         }
2903                         if (!in)
2904                                 continue;
2905                 }
2906                 if (v->gv_type == GVT_INST &&
2907                     dv->gv_type == GVT_GROUP &&
2908                     dv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2909                     !(v->gv_flags & GV_ENBLD_NOOVR))
2910                         continue;
2911 
2912                 /* Don't include excluded services and instances */
2913                 if (dv->gv_type == GVT_GROUP &&
2914                     dv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2915                         continue;
2916 
2917                 return (B_TRUE);
2918         }
2919 
2920         return (B_FALSE);
2921 }
2922 
2923 /*
2924  * Ensures that GV_INSUBGRAPH is set properly for v and its descendents.  If
2925  * any bits change, manipulate the repository appropriately.  Returns 0 or
2926  * ECONNABORTED.
2927  */
2928 static int
2929 eval_subgraph(graph_vertex_t *v, scf_handle_t *h)
2930 {
2931         boolean_t old = (v->gv_flags & GV_INSUBGRAPH) != 0;
2932         boolean_t new;
2933         graph_edge_t *e;
2934         scf_instance_t *inst;
2935         int ret = 0, r;
2936 
2937         assert(milestone != NULL && milestone != MILESTONE_NONE);
2938 
2939         new = should_be_in_subgraph(v);
2940 
2941         if (new == old)
2942                 return (0);
2943 
2944         log_framework(LOG_DEBUG, new ? "Adding %s to the subgraph.\n" :
2945             "Removing %s from the subgraph.\n", v->gv_name);
2946 
2947         v->gv_flags = (v->gv_flags & ~GV_INSUBGRAPH) |
2948             (new ? GV_INSUBGRAPH : 0);
2949 
2950         if (v->gv_type == GVT_INST && (v->gv_flags & GV_CONFIGURED)) {
2951                 int err;
2952 
2953 get_inst:
2954                 err = libscf_fmri_get_instance(h, v->gv_name, &inst);
2955                 if (err != 0) {
2956                         switch (err) {
2957                         case ECONNABORTED:
2958                                 libscf_handle_rebind(h);
2959                                 ret = ECONNABORTED;
2960                                 goto get_inst;
2961 
2962                         case ENOENT:
2963                                 break;
2964 
2965                         case EINVAL:
2966                         case ENOTSUP:
2967                         default:
2968                                 bad_error("libscf_fmri_get_instance", err);
2969                         }
2970                 } else {
2971                         const char *f;
2972 
2973                         if (new) {
2974                                 err = libscf_delete_enable_ovr(inst);
2975                                 f = "libscf_delete_enable_ovr";
2976                         } else {
2977                                 err = libscf_set_enable_ovr(inst, 0);
2978                                 f = "libscf_set_enable_ovr";
2979                         }
2980                         scf_instance_destroy(inst);
2981                         switch (err) {
2982                         case 0:
2983                         case ECANCELED:
2984                                 break;
2985 
2986                         case ECONNABORTED:
2987                                 libscf_handle_rebind(h);
2988                                 /*
2989                                  * We must continue so the graph is updated,
2990                                  * but we must return ECONNABORTED so any
2991                                  * libscf state held by any callers is reset.
2992                                  */
2993                                 ret = ECONNABORTED;
2994                                 goto get_inst;
2995 
2996                         case EROFS:
2997                         case EPERM:
2998                                 log_error(LOG_WARNING,
2999                                     "Could not set %s/%s for %s: %s.\n",
3000                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
3001                                     v->gv_name, strerror(err));
3002                                 break;
3003 
3004                         default:
3005                                 bad_error(f, err);
3006                         }
3007                 }
3008         }
3009 
3010         for (e = uu_list_first(v->gv_dependencies);
3011             e != NULL;
3012             e = uu_list_next(v->gv_dependencies, e)) {
3013                 r = eval_subgraph(e->ge_vertex, h);
3014                 if (r != 0) {
3015                         assert(r == ECONNABORTED);
3016                         ret = ECONNABORTED;
3017                 }
3018         }
3019 
3020         return (ret);
3021 }
3022 
3023 /*
3024  * Delete the (property group) dependencies of v & create new ones based on
3025  * inst.  If doing so would create a cycle, log a message and put the instance
3026  * into maintenance.  Update GV_INSUBGRAPH flags as necessary.  Returns 0 or
3027  * ECONNABORTED.
3028  */
3029 int
3030 refresh_vertex(graph_vertex_t *v, scf_instance_t *inst)
3031 {
3032         int err;
3033         int *path;
3034         char *fmri;
3035         int r;
3036         scf_handle_t *h = scf_instance_handle(inst);
3037         uu_list_t *old_deps;
3038         int ret = 0;
3039         graph_edge_t *e;
3040         graph_vertex_t *vv;
3041 
3042         assert(MUTEX_HELD(&dgraph_lock));
3043         assert(v->gv_type == GVT_INST);
3044 
3045         log_framework(LOG_DEBUG, "Graph engine: Refreshing %s.\n", v->gv_name);
3046 
3047         if (milestone > MILESTONE_NONE) {
3048                 /*
3049                  * In case some of v's dependencies are being deleted we must
3050                  * make a list of them now for GV_INSUBGRAPH-flag evaluation
3051                  * after the new dependencies are in place.
3052                  */
3053                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
3054 
3055                 err = uu_list_walk(v->gv_dependencies,
3056                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
3057                 assert(err == 0);
3058         }
3059 
3060         delete_instance_dependencies(v, B_FALSE);
3061 
3062         err = set_dependencies(v, inst, &path);
3063         switch (err) {
3064         case 0:
3065                 break;
3066 
3067         case ECONNABORTED:
3068                 ret = err;
3069                 goto out;
3070 
3071         case EINVAL:
3072         case ELOOP:
3073                 r = libscf_instance_get_fmri(inst, &fmri);
3074                 switch (r) {
3075                 case 0:
3076                         break;
3077 
3078                 case ECONNABORTED:
3079                         ret = ECONNABORTED;
3080                         goto out;
3081 
3082                 case ECANCELED:
3083                         ret = 0;
3084                         goto out;
3085 
3086                 default:
3087                         bad_error("libscf_instance_get_fmri", r);
3088                 }
3089 
3090                 if (err == EINVAL) {
3091                         log_error(LOG_ERR, "Transitioning %s "
3092                             "to maintenance due to misconfiguration.\n",
3093                             fmri ? fmri : "?");
3094                         vertex_send_event(v,
3095                             RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY);
3096                 } else {
3097                         handle_cycle(fmri, path);
3098                         vertex_send_event(v,
3099                             RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE);
3100                 }
3101                 startd_free(fmri, max_scf_fmri_size);
3102                 ret = 0;
3103                 goto out;
3104 
3105         default:
3106                 bad_error("set_dependencies", err);
3107         }
3108 
3109         if (milestone > MILESTONE_NONE) {
3110                 boolean_t aborted = B_FALSE;
3111 
3112                 for (e = uu_list_first(old_deps);
3113                     e != NULL;
3114                     e = uu_list_next(old_deps, e)) {
3115                         vv = e->ge_vertex;
3116 
3117                         if (vertex_unref(vv) == VERTEX_INUSE &&
3118                             eval_subgraph(vv, h) == ECONNABORTED)
3119                                 aborted = B_TRUE;
3120                 }
3121 
3122                 for (e = uu_list_first(v->gv_dependencies);
3123                     e != NULL;
3124                     e = uu_list_next(v->gv_dependencies, e)) {
3125                         if (eval_subgraph(e->ge_vertex, h) ==
3126                             ECONNABORTED)
3127                                 aborted = B_TRUE;
3128                 }
3129 
3130                 if (aborted) {
3131                         ret = ECONNABORTED;
3132                         goto out;
3133                 }
3134         }
3135 
3136         graph_start_if_satisfied(v);
3137 
3138         ret = 0;
3139 
3140 out:
3141         if (milestone > MILESTONE_NONE) {
3142                 void *cookie = NULL;
3143 
3144                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL)
3145                         startd_free(e, sizeof (*e));
3146 
3147                 uu_list_destroy(old_deps);
3148         }
3149 
3150         return (ret);
3151 }
3152 
3153 /*
3154  * Set up v according to inst.  That is, make sure it depends on its
3155  * restarter and set up its dependencies.  Send the ADD_INSTANCE command to
3156  * the restarter, and send ENABLE or DISABLE as appropriate.
3157  *
3158  * Returns 0 on success, ECONNABORTED on repository disconnection, or
3159  * ECANCELED if inst is deleted.
3160  */
3161 static int
3162 configure_vertex(graph_vertex_t *v, scf_instance_t *inst)
3163 {
3164         scf_handle_t *h;
3165         scf_propertygroup_t *pg;
3166         scf_snapshot_t *snap;
3167         char *restarter_fmri = startd_alloc(max_scf_value_size);
3168         int enabled, enabled_ovr;
3169         int err;
3170         int *path;
3171         int deathrow;
3172         int32_t tset;
3173 
3174         restarter_fmri[0] = '\0';
3175 
3176         assert(MUTEX_HELD(&dgraph_lock));
3177         assert(v->gv_type == GVT_INST);
3178         assert((v->gv_flags & GV_CONFIGURED) == 0);
3179 
3180         /* GV_INSUBGRAPH should already be set properly. */
3181         assert(should_be_in_subgraph(v) ==
3182             ((v->gv_flags & GV_INSUBGRAPH) != 0));
3183 
3184         /*
3185          * If the instance fmri is in the deathrow list then set the
3186          * GV_DEATHROW flag on the vertex and create and set to true the
3187          * SCF_PROPERTY_DEATHROW boolean property in the non-persistent
3188          * repository for this instance fmri.
3189          */
3190         if ((v->gv_flags & GV_DEATHROW) ||
3191             (is_fmri_in_deathrow(v->gv_name) == B_TRUE)) {
3192                 if ((v->gv_flags & GV_DEATHROW) == 0) {
3193                         /*
3194                          * Set flag GV_DEATHROW, create and set to true
3195                          * the SCF_PROPERTY_DEATHROW property in the
3196                          * non-persistent repository for this instance fmri.
3197                          */
3198                         v->gv_flags |= GV_DEATHROW;
3199 
3200                         switch (err = libscf_set_deathrow(inst, 1)) {
3201                         case 0:
3202                                 break;
3203 
3204                         case ECONNABORTED:
3205                         case ECANCELED:
3206                                 startd_free(restarter_fmri, max_scf_value_size);
3207                                 return (err);
3208 
3209                         case EROFS:
3210                                 log_error(LOG_WARNING, "Could not set %s/%s "
3211                                     "for deathrow %s: %s.\n",
3212                                     SCF_PG_DEATHROW, SCF_PROPERTY_DEATHROW,
3213                                     v->gv_name, strerror(err));
3214                                 break;
3215 
3216                         case EPERM:
3217                                 uu_die("Permission denied.\n");
3218                                 /* NOTREACHED */
3219 
3220                         default:
3221                                 bad_error("libscf_set_deathrow", err);
3222                         }
3223                         log_framework(LOG_DEBUG, "Deathrow, graph set %s.\n",
3224                             v->gv_name);
3225                 }
3226                 startd_free(restarter_fmri, max_scf_value_size);
3227                 return (0);
3228         }
3229 
3230         h = scf_instance_handle(inst);
3231 
3232         /*
3233          * Using a temporary deathrow boolean property, set through
3234          * libscf_set_deathrow(), only for fmris on deathrow, is necessary
3235          * because deathrow_fini() may already have been called, and in case
3236          * of a refresh, GV_DEATHROW may need to be set again.
3237          * libscf_get_deathrow() sets deathrow to 1 only if this instance
3238          * has a temporary boolean property named 'deathrow' valued true
3239          * in a property group 'deathrow', -1 or 0 in all other cases.
3240          */
3241         err = libscf_get_deathrow(h, inst, &deathrow);
3242         switch (err) {
3243         case 0:
3244                 break;
3245 
3246         case ECONNABORTED:
3247         case ECANCELED:
3248                 startd_free(restarter_fmri, max_scf_value_size);
3249                 return (err);
3250 
3251         default:
3252                 bad_error("libscf_get_deathrow", err);
3253         }
3254 
3255         if (deathrow == 1) {
3256                 v->gv_flags |= GV_DEATHROW;
3257                 startd_free(restarter_fmri, max_scf_value_size);
3258                 return (0);
3259         }
3260 
3261         log_framework(LOG_DEBUG, "Graph adding %s.\n", v->gv_name);
3262 
3263         /*
3264          * If the instance does not have a restarter property group,
3265          * initialize its state to uninitialized/none, in case the restarter
3266          * is not enabled.
3267          */
3268         pg = safe_scf_pg_create(h);
3269 
3270         if (scf_instance_get_pg(inst, SCF_PG_RESTARTER, pg) != 0) {
3271                 instance_data_t idata;
3272                 uint_t count = 0, msecs = ALLOC_DELAY;
3273 
3274                 switch (scf_error()) {
3275                 case SCF_ERROR_NOT_FOUND:
3276                         break;
3277 
3278                 case SCF_ERROR_CONNECTION_BROKEN:
3279                 default:
3280                         scf_pg_destroy(pg);
3281                         startd_free(restarter_fmri, max_scf_value_size);
3282                         return (ECONNABORTED);
3283 
3284                 case SCF_ERROR_DELETED:
3285                         scf_pg_destroy(pg);
3286                         startd_free(restarter_fmri, max_scf_value_size);
3287                         return (ECANCELED);
3288 
3289                 case SCF_ERROR_NOT_SET:
3290                         bad_error("scf_instance_get_pg", scf_error());
3291                 }
3292 
3293                 switch (err = libscf_instance_get_fmri(inst,
3294                     (char **)&idata.i_fmri)) {
3295                 case 0:
3296                         break;
3297 
3298                 case ECONNABORTED:
3299                 case ECANCELED:
3300                         scf_pg_destroy(pg);
3301                         startd_free(restarter_fmri, max_scf_value_size);
3302                         return (err);
3303 
3304                 default:
3305                         bad_error("libscf_instance_get_fmri", err);
3306                 }
3307 
3308                 idata.i_state = RESTARTER_STATE_NONE;
3309                 idata.i_next_state = RESTARTER_STATE_NONE;
3310 
3311 init_state:
3312                 switch (err = _restarter_commit_states(h, &idata,
3313                     RESTARTER_STATE_UNINIT, RESTARTER_STATE_NONE,
3314                     restarter_get_str_short(restarter_str_insert_in_graph))) {
3315                 case 0:
3316                         break;
3317 
3318                 case ENOMEM:
3319                         ++count;
3320                         if (count < ALLOC_RETRY) {
3321                                 (void) poll(NULL, 0, msecs);
3322                                 msecs *= ALLOC_DELAY_MULT;
3323                                 goto init_state;
3324                         }
3325 
3326                         uu_die("Insufficient memory.\n");
3327                         /* NOTREACHED */
3328 
3329                 case ECONNABORTED:
3330                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3331                         scf_pg_destroy(pg);
3332                         startd_free(restarter_fmri, max_scf_value_size);
3333                         return (ECONNABORTED);
3334 
3335                 case ENOENT:
3336                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3337                         scf_pg_destroy(pg);
3338                         startd_free(restarter_fmri, max_scf_value_size);
3339                         return (ECANCELED);
3340 
3341                 case EPERM:
3342                 case EACCES:
3343                 case EROFS:
3344                         log_error(LOG_NOTICE, "Could not initialize state for "
3345                             "%s: %s.\n", idata.i_fmri, strerror(err));
3346                         break;
3347 
3348                 case EINVAL:
3349                 default:
3350                         bad_error("_restarter_commit_states", err);
3351                 }
3352 
3353                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3354         }
3355 
3356         scf_pg_destroy(pg);
3357 
3358         if (milestone != NULL) {
3359                 /*
3360                  * Make sure the enable-override is set properly before we
3361                  * read whether we should be enabled.
3362                  */
3363                 if (milestone == MILESTONE_NONE ||
3364                     !(v->gv_flags & GV_INSUBGRAPH)) {
3365                         /*
3366                          * This might seem unjustified after the milestone
3367                          * transition has completed (non_subgraph_svcs == 0),
3368                          * but it's important because when we boot to
3369                          * a milestone, we set the milestone before populating
3370                          * the graph, and all of the new non-subgraph services
3371                          * need to be disabled here.
3372                          */
3373                         switch (err = libscf_set_enable_ovr(inst, 0)) {
3374                         case 0:
3375                                 break;
3376 
3377                         case ECONNABORTED:
3378                         case ECANCELED:
3379                                 startd_free(restarter_fmri, max_scf_value_size);
3380                                 return (err);
3381 
3382                         case EROFS:
3383                                 log_error(LOG_WARNING,
3384                                     "Could not set %s/%s for %s: %s.\n",
3385                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
3386                                     v->gv_name, strerror(err));
3387                                 break;
3388 
3389                         case EPERM:
3390                                 uu_die("Permission denied.\n");
3391                                 /* NOTREACHED */
3392 
3393                         default:
3394                                 bad_error("libscf_set_enable_ovr", err);
3395                         }
3396                 } else {
3397                         assert(v->gv_flags & GV_INSUBGRAPH);
3398                         switch (err = libscf_delete_enable_ovr(inst)) {
3399                         case 0:
3400                                 break;
3401 
3402                         case ECONNABORTED:
3403                         case ECANCELED:
3404                                 startd_free(restarter_fmri, max_scf_value_size);
3405                                 return (err);
3406 
3407                         case EPERM:
3408                                 uu_die("Permission denied.\n");
3409                                 /* NOTREACHED */
3410 
3411                         default:
3412                                 bad_error("libscf_delete_enable_ovr", err);
3413                         }
3414                 }
3415         }
3416 
3417         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
3418             &enabled_ovr, &restarter_fmri);
3419         switch (err) {
3420         case 0:
3421                 break;
3422 
3423         case ECONNABORTED:
3424         case ECANCELED:
3425                 startd_free(restarter_fmri, max_scf_value_size);
3426                 return (err);
3427 
3428         case ENOENT:
3429                 log_framework(LOG_DEBUG,
3430                     "Ignoring %s because it has no general property group.\n",
3431                     v->gv_name);
3432                 startd_free(restarter_fmri, max_scf_value_size);
3433                 return (0);
3434 
3435         default:
3436                 bad_error("libscf_get_basic_instance_data", err);
3437         }
3438 
3439         if ((tset = libscf_get_stn_tset(inst)) == -1) {
3440                 log_framework(LOG_WARNING,
3441                     "Failed to get notification parameters for %s: %s\n",
3442                     v->gv_name, scf_strerror(scf_error()));
3443                 v->gv_stn_tset = 0;
3444         } else {
3445                 v->gv_stn_tset = tset;
3446         }
3447         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
3448                 stn_global = v->gv_stn_tset;
3449 
3450         if (enabled == -1) {
3451                 startd_free(restarter_fmri, max_scf_value_size);
3452                 return (0);
3453         }
3454 
3455         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
3456             (enabled ? GV_ENBLD_NOOVR : 0);
3457 
3458         if (enabled_ovr != -1)
3459                 enabled = enabled_ovr;
3460 
3461         v->gv_state = RESTARTER_STATE_UNINIT;
3462 
3463         snap = libscf_get_or_make_running_snapshot(inst, v->gv_name, B_TRUE);
3464         scf_snapshot_destroy(snap);
3465 
3466         /* Set up the restarter. (Sends _ADD_INSTANCE on success.) */
3467         err = graph_change_restarter(v, restarter_fmri, h, &path);
3468         if (err != 0) {
3469                 instance_data_t idata;
3470                 uint_t count = 0, msecs = ALLOC_DELAY;
3471                 restarter_str_t reason;
3472 
3473                 if (err == ECONNABORTED) {
3474                         startd_free(restarter_fmri, max_scf_value_size);
3475                         return (err);
3476                 }
3477 
3478                 assert(err == EINVAL || err == ELOOP);
3479 
3480                 if (err == EINVAL) {
3481                         log_framework(LOG_ERR, emsg_invalid_restarter,
3482                             v->gv_name, restarter_fmri);
3483                         reason = restarter_str_invalid_restarter;
3484                 } else {
3485                         handle_cycle(v->gv_name, path);
3486                         reason = restarter_str_dependency_cycle;
3487                 }
3488 
3489                 startd_free(restarter_fmri, max_scf_value_size);
3490 
3491                 /*
3492                  * We didn't register the instance with the restarter, so we
3493                  * must set maintenance mode ourselves.
3494                  */
3495                 err = libscf_instance_get_fmri(inst, (char **)&idata.i_fmri);
3496                 if (err != 0) {
3497                         assert(err == ECONNABORTED || err == ECANCELED);
3498                         return (err);
3499                 }
3500 
3501                 idata.i_state = RESTARTER_STATE_NONE;
3502                 idata.i_next_state = RESTARTER_STATE_NONE;
3503 
3504 set_maint:
3505                 switch (err = _restarter_commit_states(h, &idata,
3506                     RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE,
3507                     restarter_get_str_short(reason))) {
3508                 case 0:
3509                         break;
3510 
3511                 case ENOMEM:
3512                         ++count;
3513                         if (count < ALLOC_RETRY) {
3514                                 (void) poll(NULL, 0, msecs);
3515                                 msecs *= ALLOC_DELAY_MULT;
3516                                 goto set_maint;
3517                         }
3518 
3519                         uu_die("Insufficient memory.\n");
3520                         /* NOTREACHED */
3521 
3522                 case ECONNABORTED:
3523                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3524                         return (ECONNABORTED);
3525 
3526                 case ENOENT:
3527                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3528                         return (ECANCELED);
3529 
3530                 case EPERM:
3531                 case EACCES:
3532                 case EROFS:
3533                         log_error(LOG_NOTICE, "Could not initialize state for "
3534                             "%s: %s.\n", idata.i_fmri, strerror(err));
3535                         break;
3536 
3537                 case EINVAL:
3538                 default:
3539                         bad_error("_restarter_commit_states", err);
3540                 }
3541 
3542                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3543 
3544                 v->gv_state = RESTARTER_STATE_MAINT;
3545 
3546                 goto out;
3547         }
3548         startd_free(restarter_fmri, max_scf_value_size);
3549 
3550         /* Add all the other dependencies. */
3551         err = refresh_vertex(v, inst);
3552         if (err != 0) {
3553                 assert(err == ECONNABORTED);
3554                 return (err);
3555         }
3556 
3557 out:
3558         v->gv_flags |= GV_CONFIGURED;
3559 
3560         graph_enable_by_vertex(v, enabled, 0);
3561 
3562         return (0);
3563 }
3564 
3565 
3566 static void
3567 kill_user_procs(void)
3568 {
3569         (void) fputs("svc.startd: Killing user processes.\n", stdout);
3570 
3571         /*
3572          * Despite its name, killall's role is to get select user processes--
3573          * basically those representing terminal-based logins-- to die.  Victims
3574          * are located by killall in the utmp database.  Since these are most
3575          * often shell based logins, and many shells mask SIGTERM (but are
3576          * responsive to SIGHUP) we first HUP and then shortly thereafter
3577          * kill -9.
3578          */
3579         (void) fork_with_timeout("/usr/sbin/killall HUP", 1, 5);
3580         (void) fork_with_timeout("/usr/sbin/killall KILL", 1, 5);
3581 
3582         /*
3583          * Note the selection of user id's 0, 1 and 15, subsequently
3584          * inverted by -v.  15 is reserved for dladmd.  Yes, this is a
3585          * kludge-- a better policy is needed.
3586          *
3587          * Note that fork_with_timeout will only wait out the 1 second
3588          * "grace time" if pkill actually returns 0.  So if there are
3589          * no matches, this will run to completion much more quickly.
3590          */
3591         (void) fork_with_timeout("/usr/bin/pkill -TERM -v -u 0,1,15", 1, 5);
3592         (void) fork_with_timeout("/usr/bin/pkill -KILL -v -u 0,1,15", 1, 5);
3593 }
3594 
3595 static void
3596 do_uadmin(void)
3597 {
3598         const char * const resetting = "/etc/svc/volatile/resetting";
3599         int fd;
3600         struct statvfs vfs;
3601         time_t now;
3602         struct tm nowtm;
3603         char down_buf[256], time_buf[256];
3604         uintptr_t mdep;
3605 #if defined(__i386)
3606         grub_boot_args_t fbarg;
3607 #endif  /* __i386 */
3608 
3609         mdep = NULL;
3610         fd = creat(resetting, 0777);
3611         if (fd >= 0)
3612                 startd_close(fd);
3613         else
3614                 uu_warn("Could not create \"%s\"", resetting);
3615 
3616         /* Kill dhcpagent if we're not using nfs for root */
3617         if ((statvfs("/", &vfs) == 0) &&
3618             (strncmp(vfs.f_basetype, "nfs", sizeof ("nfs") - 1) != 0))
3619                 fork_with_timeout("/usr/bin/pkill -x -u 0 dhcpagent", 0, 5);
3620 
3621         /*
3622          * Call sync(2) now, before we kill off user processes.  This takes
3623          * advantage of the several seconds of pause we have before the
3624          * killalls are done.  Time we can make good use of to get pages
3625          * moving out to disk.
3626          *
3627          * Inside non-global zones, we don't bother, and it's better not to
3628          * anyway, since sync(2) can have system-wide impact.
3629          */
3630         if (getzoneid() == 0)
3631                 sync();
3632 
3633         kill_user_procs();
3634 
3635         /*
3636          * Note that this must come after the killing of user procs, since
3637          * killall relies on utmpx, and this command affects the contents of
3638          * said file.
3639          */
3640         if (access("/usr/lib/acct/closewtmp", X_OK) == 0)
3641                 fork_with_timeout("/usr/lib/acct/closewtmp", 0, 5);
3642 
3643         /*
3644          * For patches which may be installed as the system is shutting
3645          * down, we need to ensure, one more time, that the boot archive
3646          * really is up to date.
3647          */
3648         if (getzoneid() == 0 && access("/usr/sbin/bootadm", X_OK) == 0)
3649                 fork_with_timeout("/usr/sbin/bootadm -ea update_all", 0, 3600);
3650 
3651         /*
3652          * Right now, fast reboot is supported only on i386.
3653          * scf_is_fastboot_default() should take care of it.
3654          * If somehow we got there on unsupported platform -
3655          * print warning and fall back to regular reboot.
3656          */
3657         if (halting == AD_FASTREBOOT) {
3658 #if defined(__i386)
3659                 int rc;
3660 
3661                 if ((rc = grub_get_boot_args(&fbarg, NULL,
3662                     GRUB_ENTRY_DEFAULT)) == 0) {
3663                         mdep = (uintptr_t)&fbarg.gba_bootargs;
3664                 } else {
3665                         /*
3666                          * Failed to read GRUB menu, fall back to normal reboot
3667                          */
3668                         halting = AD_BOOT;
3669                         uu_warn("Failed to process GRUB menu entry "
3670                             "for fast reboot.\n\t%s\n"
3671                             "Falling back to regular reboot.\n",
3672                             grub_strerror(rc));
3673                 }
3674 #else   /* __i386 */
3675                 halting = AD_BOOT;
3676                 uu_warn("Fast reboot configured, but not supported by "
3677                     "this ISA\n");
3678 #endif  /* __i386 */
3679         }
3680 
3681         fork_with_timeout("/sbin/umountall -l", 0, 5);
3682         fork_with_timeout("/sbin/umount /tmp /var/adm /var/run /var "
3683             ">/dev/null 2>&1", 0, 5);
3684 
3685         /*
3686          * Try to get to consistency for whatever UFS filesystems are left.
3687          * This is pretty expensive, so we save it for the end in the hopes of
3688          * minimizing what it must do.  The other option would be to start in
3689          * parallel with the killall's, but lockfs tends to throw out much more
3690          * than is needed, and so subsequent commands (like umountall) take a
3691          * long time to get going again.
3692          *
3693          * Inside of zones, we don't bother, since we're not about to terminate
3694          * the whole OS instance.
3695          *
3696          * On systems using only ZFS, this call to lockfs -fa is a no-op.
3697          */
3698         if (getzoneid() == 0) {
3699                 if (access("/usr/sbin/lockfs", X_OK) == 0)
3700                         fork_with_timeout("/usr/sbin/lockfs -fa", 0, 30);
3701 
3702                 sync(); /* once more, with feeling */
3703         }
3704 
3705         fork_with_timeout("/sbin/umount /usr >/dev/null 2>&1", 0, 5);
3706 
3707         /*
3708          * Construct and emit the last words from userland:
3709          * "<timestamp> The system is down.  Shutdown took <N> seconds."
3710          *
3711          * Normally we'd use syslog, but with /var and other things
3712          * potentially gone, try to minimize the external dependencies.
3713          */
3714         now = time(NULL);
3715         (void) localtime_r(&now, &nowtm);
3716 
3717         if (strftime(down_buf, sizeof (down_buf),
3718             "%b %e %T The system is down.", &nowtm) == 0) {
3719                 (void) strlcpy(down_buf, "The system is down.",
3720                     sizeof (down_buf));
3721         }
3722 
3723         if (halting_time != 0 && halting_time <= now) {
3724                 (void) snprintf(time_buf, sizeof (time_buf),
3725                     "  Shutdown took %lu seconds.", now - halting_time);
3726         } else {
3727                 time_buf[0] = '\0';
3728         }
3729         (void) printf("%s%s\n", down_buf, time_buf);
3730 
3731         (void) uadmin(A_SHUTDOWN, halting, mdep);
3732         uu_warn("uadmin() failed");
3733 
3734 #if defined(__i386)
3735         /* uadmin fail, cleanup grub_boot_args */
3736         if (halting == AD_FASTREBOOT)
3737                 grub_cleanup_boot_args(&fbarg);
3738 #endif  /* __i386 */
3739 
3740         if (remove(resetting) != 0 && errno != ENOENT)
3741                 uu_warn("Could not remove \"%s\"", resetting);
3742 }
3743 
3744 /*
3745  * If any of the up_svcs[] are online or satisfiable, return true.  If they are
3746  * all missing, disabled, in maintenance, or unsatisfiable, return false.
3747  */
3748 boolean_t
3749 can_come_up(void)
3750 {
3751         int i;
3752 
3753         assert(MUTEX_HELD(&dgraph_lock));
3754 
3755         /*
3756          * If we are booting to single user (boot -s),
3757          * SCF_MILESTONE_SINGLE_USER is needed to come up because startd
3758          * spawns sulogin after single-user is online (see specials.c).
3759          */
3760         i = (booting_to_single_user ? 0 : 1);
3761 
3762         for (; up_svcs[i] != NULL; ++i) {
3763                 if (up_svcs_p[i] == NULL) {
3764                         up_svcs_p[i] = vertex_get_by_name(up_svcs[i]);
3765 
3766                         if (up_svcs_p[i] == NULL)
3767                                 continue;
3768                 }
3769 
3770                 /*
3771                  * Ignore unconfigured services (the ones that have been
3772                  * mentioned in a dependency from other services, but do
3773                  * not exist in the repository).  Services which exist
3774                  * in the repository but don't have general/enabled
3775                  * property will be also ignored.
3776                  */
3777                 if (!(up_svcs_p[i]->gv_flags & GV_CONFIGURED))
3778                         continue;
3779 
3780                 switch (up_svcs_p[i]->gv_state) {
3781                 case RESTARTER_STATE_ONLINE:
3782                 case RESTARTER_STATE_DEGRADED:
3783                         /*
3784                          * Deactivate verbose boot once a login service has been
3785                          * reached.
3786                          */
3787                         st->st_log_login_reached = 1;
3788                         /*FALLTHROUGH*/
3789                 case RESTARTER_STATE_UNINIT:
3790                         return (B_TRUE);
3791 
3792                 case RESTARTER_STATE_OFFLINE:
3793                         if (instance_satisfied(up_svcs_p[i], B_TRUE) != -1)
3794                                 return (B_TRUE);
3795                         log_framework(LOG_DEBUG,
3796                             "can_come_up(): %s is unsatisfiable.\n",
3797                             up_svcs_p[i]->gv_name);
3798                         continue;
3799 
3800                 case RESTARTER_STATE_DISABLED:
3801                 case RESTARTER_STATE_MAINT:
3802                         log_framework(LOG_DEBUG,
3803                             "can_come_up(): %s is in state %s.\n",
3804                             up_svcs_p[i]->gv_name,
3805                             instance_state_str[up_svcs_p[i]->gv_state]);
3806                         continue;
3807 
3808                 default:
3809 #ifndef NDEBUG
3810                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
3811                             __FILE__, __LINE__, up_svcs_p[i]->gv_state);
3812 #endif
3813                         abort();
3814                 }
3815         }
3816 
3817         /*
3818          * In the seed repository, console-login is unsatisfiable because
3819          * services are missing.  To behave correctly in that case we don't want
3820          * to return false until manifest-import is online.
3821          */
3822 
3823         if (manifest_import_p == NULL) {
3824                 manifest_import_p = vertex_get_by_name(manifest_import);
3825 
3826                 if (manifest_import_p == NULL)
3827                         return (B_FALSE);
3828         }
3829 
3830         switch (manifest_import_p->gv_state) {
3831         case RESTARTER_STATE_ONLINE:
3832         case RESTARTER_STATE_DEGRADED:
3833         case RESTARTER_STATE_DISABLED:
3834         case RESTARTER_STATE_MAINT:
3835                 break;
3836 
3837         case RESTARTER_STATE_OFFLINE:
3838                 if (instance_satisfied(manifest_import_p, B_TRUE) == -1)
3839                         break;
3840                 /* FALLTHROUGH */
3841 
3842         case RESTARTER_STATE_UNINIT:
3843                 return (B_TRUE);
3844         }
3845 
3846         return (B_FALSE);
3847 }
3848 
3849 /*
3850  * Runs sulogin.  Returns
3851  *   0 - success
3852  *   EALREADY - sulogin is already running
3853  *   EBUSY - console-login is running
3854  */
3855 static int
3856 run_sulogin(const char *msg)
3857 {
3858         graph_vertex_t *v;
3859 
3860         assert(MUTEX_HELD(&dgraph_lock));
3861 
3862         if (sulogin_running)
3863                 return (EALREADY);
3864 
3865         v = vertex_get_by_name(console_login_fmri);
3866         if (v != NULL && inst_running(v))
3867                 return (EBUSY);
3868 
3869         sulogin_running = B_TRUE;
3870 
3871         MUTEX_UNLOCK(&dgraph_lock);
3872 
3873         fork_sulogin(B_FALSE, msg);
3874 
3875         MUTEX_LOCK(&dgraph_lock);
3876 
3877         sulogin_running = B_FALSE;
3878 
3879         if (console_login_ready) {
3880                 v = vertex_get_by_name(console_login_fmri);
3881 
3882                 if (v != NULL && v->gv_state == RESTARTER_STATE_OFFLINE) {
3883                         if (v->gv_start_f == NULL)
3884                                 vertex_send_event(v,
3885                                     RESTARTER_EVENT_TYPE_START);
3886                         else
3887                                 v->gv_start_f(v);
3888                 }
3889 
3890                 console_login_ready = B_FALSE;
3891         }
3892 
3893         return (0);
3894 }
3895 
3896 /*
3897  * The sulogin thread runs sulogin while can_come_up() is false.  run_sulogin()
3898  * keeps sulogin from stepping on console-login's toes.
3899  */
3900 /* ARGSUSED */
3901 static void *
3902 sulogin_thread(void *unused)
3903 {
3904         MUTEX_LOCK(&dgraph_lock);
3905 
3906         assert(sulogin_thread_running);
3907 
3908         do {
3909                 (void) run_sulogin("Console login service(s) cannot run\n");
3910         } while (!can_come_up());
3911 
3912         sulogin_thread_running = B_FALSE;
3913         MUTEX_UNLOCK(&dgraph_lock);
3914 
3915         return (NULL);
3916 }
3917 
3918 /* ARGSUSED */
3919 void *
3920 single_user_thread(void *unused)
3921 {
3922         uint_t left;
3923         scf_handle_t *h;
3924         scf_instance_t *inst;
3925         scf_property_t *prop;
3926         scf_value_t *val;
3927         const char *msg;
3928         char *buf;
3929         int r;
3930 
3931         MUTEX_LOCK(&single_user_thread_lock);
3932         single_user_thread_count++;
3933 
3934         if (!booting_to_single_user)
3935                 kill_user_procs();
3936 
3937         if (go_single_user_mode || booting_to_single_user) {
3938                 msg = "SINGLE USER MODE\n";
3939         } else {
3940                 assert(go_to_level1);
3941 
3942                 fork_rc_script('1', "start", B_TRUE);
3943 
3944                 uu_warn("The system is ready for administration.\n");
3945 
3946                 msg = "";
3947         }
3948 
3949         MUTEX_UNLOCK(&single_user_thread_lock);
3950 
3951         for (;;) {
3952                 MUTEX_LOCK(&dgraph_lock);
3953                 r = run_sulogin(msg);
3954                 MUTEX_UNLOCK(&dgraph_lock);
3955                 if (r == 0)
3956                         break;
3957 
3958                 assert(r == EALREADY || r == EBUSY);
3959 
3960                 left = 3;
3961                 while (left > 0)
3962                         left = sleep(left);
3963         }
3964 
3965         MUTEX_LOCK(&single_user_thread_lock);
3966 
3967         /*
3968          * If another single user thread has started, let it finish changing
3969          * the run level.
3970          */
3971         if (single_user_thread_count > 1) {
3972                 single_user_thread_count--;
3973                 MUTEX_UNLOCK(&single_user_thread_lock);
3974                 return (NULL);
3975         }
3976 
3977         h = libscf_handle_create_bound_loop();
3978         inst = scf_instance_create(h);
3979         prop = safe_scf_property_create(h);
3980         val = safe_scf_value_create(h);
3981         buf = startd_alloc(max_scf_fmri_size);
3982 
3983 lookup:
3984         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
3985             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
3986                 switch (scf_error()) {
3987                 case SCF_ERROR_NOT_FOUND:
3988                         r = libscf_create_self(h);
3989                         if (r == 0)
3990                                 goto lookup;
3991                         assert(r == ECONNABORTED);
3992                         /* FALLTHROUGH */
3993 
3994                 case SCF_ERROR_CONNECTION_BROKEN:
3995                         libscf_handle_rebind(h);
3996                         goto lookup;
3997 
3998                 case SCF_ERROR_INVALID_ARGUMENT:
3999                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4000                 case SCF_ERROR_NOT_BOUND:
4001                 case SCF_ERROR_HANDLE_MISMATCH:
4002                 default:
4003                         bad_error("scf_handle_decode_fmri", scf_error());
4004                 }
4005         }
4006 
4007         MUTEX_LOCK(&dgraph_lock);
4008 
4009         r = scf_instance_delete_prop(inst, SCF_PG_OPTIONS_OVR,
4010             SCF_PROPERTY_MILESTONE);
4011         switch (r) {
4012         case 0:
4013         case ECANCELED:
4014                 break;
4015 
4016         case ECONNABORTED:
4017                 MUTEX_UNLOCK(&dgraph_lock);
4018                 libscf_handle_rebind(h);
4019                 goto lookup;
4020 
4021         case EPERM:
4022         case EACCES:
4023         case EROFS:
4024                 log_error(LOG_WARNING, "Could not clear temporary milestone: "
4025                     "%s.\n", strerror(r));
4026                 break;
4027 
4028         default:
4029                 bad_error("scf_instance_delete_prop", r);
4030         }
4031 
4032         MUTEX_UNLOCK(&dgraph_lock);
4033 
4034         r = libscf_get_milestone(inst, prop, val, buf, max_scf_fmri_size);
4035         switch (r) {
4036         case ECANCELED:
4037         case ENOENT:
4038         case EINVAL:
4039                 (void) strcpy(buf, "all");
4040                 /* FALLTHROUGH */
4041 
4042         case 0:
4043                 uu_warn("Returning to milestone %s.\n", buf);
4044                 break;
4045 
4046         case ECONNABORTED:
4047                 libscf_handle_rebind(h);
4048                 goto lookup;
4049 
4050         default:
4051                 bad_error("libscf_get_milestone", r);
4052         }
4053 
4054         r = dgraph_set_milestone(buf, h, B_FALSE);
4055         switch (r) {
4056         case 0:
4057         case ECONNRESET:
4058         case EALREADY:
4059         case EINVAL:
4060         case ENOENT:
4061                 break;
4062 
4063         default:
4064                 bad_error("dgraph_set_milestone", r);
4065         }
4066 
4067         /*
4068          * See graph_runlevel_changed().
4069          */
4070         MUTEX_LOCK(&dgraph_lock);
4071         utmpx_set_runlevel(target_milestone_as_runlevel(), 'S', B_TRUE);
4072         MUTEX_UNLOCK(&dgraph_lock);
4073 
4074         startd_free(buf, max_scf_fmri_size);
4075         scf_value_destroy(val);
4076         scf_property_destroy(prop);
4077         scf_instance_destroy(inst);
4078         scf_handle_destroy(h);
4079 
4080         /*
4081          * We'll give ourselves 3 seconds to respond to all of the enablings
4082          * that setting the milestone should have created before checking
4083          * whether to run sulogin.
4084          */
4085         left = 3;
4086         while (left > 0)
4087                 left = sleep(left);
4088 
4089         MUTEX_LOCK(&dgraph_lock);
4090         /*
4091          * Clearing these variables will allow the sulogin thread to run.  We
4092          * check here in case there aren't any more state updates anytime soon.
4093          */
4094         go_to_level1 = go_single_user_mode = booting_to_single_user = B_FALSE;
4095         if (!sulogin_thread_running && !can_come_up()) {
4096                 (void) startd_thread_create(sulogin_thread, NULL);
4097                 sulogin_thread_running = B_TRUE;
4098         }
4099         MUTEX_UNLOCK(&dgraph_lock);
4100         single_user_thread_count--;
4101         MUTEX_UNLOCK(&single_user_thread_lock);
4102         return (NULL);
4103 }
4104 
4105 
4106 /*
4107  * Dependency graph operations API.  These are handle-independent thread-safe
4108  * graph manipulation functions which are the entry points for the event
4109  * threads below.
4110  */
4111 
4112 /*
4113  * If a configured vertex exists for inst_fmri, return EEXIST.  If no vertex
4114  * exists for inst_fmri, add one.  Then fetch the restarter from inst, make
4115  * this vertex dependent on it, and send _ADD_INSTANCE to the restarter.
4116  * Fetch whether the instance should be enabled from inst and send _ENABLE or
4117  * _DISABLE as appropriate.  Finally rummage through inst's dependency
4118  * property groups and add vertices and edges as appropriate.  If anything
4119  * goes wrong after sending _ADD_INSTANCE, send _ADMIN_MAINT_ON to put the
4120  * instance in maintenance.  Don't send _START or _STOP until we get a state
4121  * update in case we're being restarted and the service is already running.
4122  *
4123  * To support booting to a milestone, we must also make sure all dependencies
4124  * encountered are configured, if they exist in the repository.
4125  *
4126  * Returns 0 on success, ECONNABORTED on repository disconnection, EINVAL if
4127  * inst_fmri is an invalid (or not canonical) FMRI, ECANCELED if inst is
4128  * deleted, or EEXIST if a configured vertex for inst_fmri already exists.
4129  */
4130 int
4131 dgraph_add_instance(const char *inst_fmri, scf_instance_t *inst,
4132     boolean_t lock_graph)
4133 {
4134         graph_vertex_t *v;
4135         int err;
4136 
4137         if (strcmp(inst_fmri, SCF_SERVICE_STARTD) == 0)
4138                 return (0);
4139 
4140         /* Check for a vertex for inst_fmri. */
4141         if (lock_graph) {
4142                 MUTEX_LOCK(&dgraph_lock);
4143         } else {
4144                 assert(MUTEX_HELD(&dgraph_lock));
4145         }
4146 
4147         v = vertex_get_by_name(inst_fmri);
4148 
4149         if (v != NULL) {
4150                 assert(v->gv_type == GVT_INST);
4151 
4152                 if (v->gv_flags & GV_CONFIGURED) {
4153                         if (lock_graph)
4154                                 MUTEX_UNLOCK(&dgraph_lock);
4155                         return (EEXIST);
4156                 }
4157         } else {
4158                 /* Add the vertex. */
4159                 err = graph_insert_vertex_unconfigured(inst_fmri, GVT_INST, 0,
4160                     RERR_NONE, &v);
4161                 if (err != 0) {
4162                         assert(err == EINVAL);
4163                         if (lock_graph)
4164                                 MUTEX_UNLOCK(&dgraph_lock);
4165                         return (EINVAL);
4166                 }
4167         }
4168 
4169         err = configure_vertex(v, inst);
4170 
4171         if (lock_graph)
4172                 MUTEX_UNLOCK(&dgraph_lock);
4173 
4174         return (err);
4175 }
4176 
4177 /*
4178  * Locate the vertex for this property group's instance.  If it doesn't exist
4179  * or is unconfigured, call dgraph_add_instance() & return.  Otherwise fetch
4180  * the restarter for the instance, and if it has changed, send
4181  * _REMOVE_INSTANCE to the old restarter, remove the dependency, make sure the
4182  * new restarter has a vertex, add a new dependency, and send _ADD_INSTANCE to
4183  * the new restarter.  Then fetch whether the instance should be enabled, and
4184  * if it is different from what we had, or if we changed the restarter, send
4185  * the appropriate _ENABLE or _DISABLE command.
4186  *
4187  * Returns 0 on success, ENOTSUP if the pg's parent is not an instance,
4188  * ECONNABORTED on repository disconnection, ECANCELED if the instance is
4189  * deleted, or -1 if the instance's general property group is deleted or if
4190  * its enabled property is misconfigured.
4191  */
4192 static int
4193 dgraph_update_general(scf_propertygroup_t *pg)
4194 {
4195         scf_handle_t *h;
4196         scf_instance_t *inst;
4197         char *fmri;
4198         char *restarter_fmri;
4199         graph_vertex_t *v;
4200         int err;
4201         int enabled, enabled_ovr;
4202         int oldflags;
4203 
4204         /* Find the vertex for this service */
4205         h = scf_pg_handle(pg);
4206 
4207         inst = safe_scf_instance_create(h);
4208 
4209         if (scf_pg_get_parent_instance(pg, inst) != 0) {
4210                 switch (scf_error()) {
4211                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4212                         return (ENOTSUP);
4213 
4214                 case SCF_ERROR_CONNECTION_BROKEN:
4215                 default:
4216                         return (ECONNABORTED);
4217 
4218                 case SCF_ERROR_DELETED:
4219                         return (0);
4220 
4221                 case SCF_ERROR_NOT_SET:
4222                         bad_error("scf_pg_get_parent_instance", scf_error());
4223                 }
4224         }
4225 
4226         err = libscf_instance_get_fmri(inst, &fmri);
4227         switch (err) {
4228         case 0:
4229                 break;
4230 
4231         case ECONNABORTED:
4232                 scf_instance_destroy(inst);
4233                 return (ECONNABORTED);
4234 
4235         case ECANCELED:
4236                 scf_instance_destroy(inst);
4237                 return (0);
4238 
4239         default:
4240                 bad_error("libscf_instance_get_fmri", err);
4241         }
4242 
4243         log_framework(LOG_DEBUG,
4244             "Graph engine: Reloading general properties for %s.\n", fmri);
4245 
4246         MUTEX_LOCK(&dgraph_lock);
4247 
4248         v = vertex_get_by_name(fmri);
4249         if (v == NULL || !(v->gv_flags & GV_CONFIGURED)) {
4250                 /* Will get the up-to-date properties. */
4251                 MUTEX_UNLOCK(&dgraph_lock);
4252                 err = dgraph_add_instance(fmri, inst, B_TRUE);
4253                 startd_free(fmri, max_scf_fmri_size);
4254                 scf_instance_destroy(inst);
4255                 return (err == ECANCELED ? 0 : err);
4256         }
4257 
4258         /* Read enabled & restarter from repository. */
4259         restarter_fmri = startd_alloc(max_scf_value_size);
4260         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
4261             &enabled_ovr, &restarter_fmri);
4262         if (err != 0 || enabled == -1) {
4263                 MUTEX_UNLOCK(&dgraph_lock);
4264                 scf_instance_destroy(inst);
4265                 startd_free(fmri, max_scf_fmri_size);
4266 
4267                 switch (err) {
4268                 case ENOENT:
4269                 case 0:
4270                         startd_free(restarter_fmri, max_scf_value_size);
4271                         return (-1);
4272 
4273                 case ECONNABORTED:
4274                 case ECANCELED:
4275                         startd_free(restarter_fmri, max_scf_value_size);
4276                         return (err);
4277 
4278                 default:
4279                         bad_error("libscf_get_basic_instance_data", err);
4280                 }
4281         }
4282 
4283         oldflags = v->gv_flags;
4284         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
4285             (enabled ? GV_ENBLD_NOOVR : 0);
4286 
4287         if (enabled_ovr != -1)
4288                 enabled = enabled_ovr;
4289 
4290         /*
4291          * If GV_ENBLD_NOOVR has changed, then we need to re-evaluate the
4292          * subgraph.
4293          */
4294         if (milestone > MILESTONE_NONE && v->gv_flags != oldflags)
4295                 (void) eval_subgraph(v, h);
4296 
4297         scf_instance_destroy(inst);
4298 
4299         /* Ignore restarter change for now. */
4300 
4301         startd_free(restarter_fmri, max_scf_value_size);
4302         startd_free(fmri, max_scf_fmri_size);
4303 
4304         /*
4305          * Always send _ENABLE or _DISABLE.  We could avoid this if the
4306          * restarter didn't change and the enabled value didn't change, but
4307          * that's not easy to check and improbable anyway, so we'll just do
4308          * this.
4309          */
4310         graph_enable_by_vertex(v, enabled, 1);
4311 
4312         MUTEX_UNLOCK(&dgraph_lock);
4313 
4314         return (0);
4315 }
4316 
4317 /*
4318  * Delete all of the property group dependencies of v, update inst's running
4319  * snapshot, and add the dependencies in the new snapshot.  If any of the new
4320  * dependencies would create a cycle, send _ADMIN_MAINT_ON.  Otherwise
4321  * reevaluate v's dependencies, send _START or _STOP as appropriate, and do
4322  * the same for v's dependents.
4323  *
4324  * Returns
4325  *   0 - success
4326  *   ECONNABORTED - repository connection broken
4327  *   ECANCELED - inst was deleted
4328  *   EINVAL - inst is invalid (e.g., missing general/enabled)
4329  *   -1 - libscf_snapshots_refresh() failed
4330  */
4331 static int
4332 dgraph_refresh_instance(graph_vertex_t *v, scf_instance_t *inst)
4333 {
4334         int r;
4335         int enabled;
4336         int32_t tset;
4337 
4338         assert(MUTEX_HELD(&dgraph_lock));
4339         assert(v->gv_type == GVT_INST);
4340 
4341         /* Only refresh services with valid general/enabled properties. */
4342         r = libscf_get_basic_instance_data(scf_instance_handle(inst), inst,
4343             v->gv_name, &enabled, NULL, NULL);
4344         switch (r) {
4345         case 0:
4346                 break;
4347 
4348         case ECONNABORTED:
4349         case ECANCELED:
4350                 return (r);
4351 
4352         case ENOENT:
4353                 log_framework(LOG_DEBUG,
4354                     "Ignoring %s because it has no general property group.\n",
4355                     v->gv_name);
4356                 return (EINVAL);
4357 
4358         default:
4359                 bad_error("libscf_get_basic_instance_data", r);
4360         }
4361 
4362         if ((tset = libscf_get_stn_tset(inst)) == -1) {
4363                 log_framework(LOG_WARNING,
4364                     "Failed to get notification parameters for %s: %s\n",
4365                     v->gv_name, scf_strerror(scf_error()));
4366                 tset = 0;
4367         }
4368         v->gv_stn_tset = tset;
4369         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
4370                 stn_global = tset;
4371 
4372         if (enabled == -1)
4373                 return (EINVAL);
4374 
4375         r = libscf_snapshots_refresh(inst, v->gv_name);
4376         if (r != 0) {
4377                 if (r != -1)
4378                         bad_error("libscf_snapshots_refresh", r);
4379 
4380                 /* error logged */
4381                 return (r);
4382         }
4383 
4384         r = refresh_vertex(v, inst);
4385         if (r != 0 && r != ECONNABORTED)
4386                 bad_error("refresh_vertex", r);
4387         return (r);
4388 }
4389 
4390 /*
4391  * Returns true only if none of this service's dependents are 'up' -- online
4392  * or degraded (offline is considered down in this situation). This function
4393  * is somehow similar to is_nonsubgraph_leaf() but works on subtrees.
4394  */
4395 static boolean_t
4396 insubtree_dependents_down(graph_vertex_t *v)
4397 {
4398         graph_vertex_t *vv;
4399         graph_edge_t *e;
4400 
4401         assert(MUTEX_HELD(&dgraph_lock));
4402 
4403         for (e = uu_list_first(v->gv_dependents); e != NULL;
4404             e = uu_list_next(v->gv_dependents, e)) {
4405                 vv = e->ge_vertex;
4406                 if (vv->gv_type == GVT_INST) {
4407                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4408                                 continue;
4409 
4410                         if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4411                                 continue;
4412 
4413                         if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4414                             (vv->gv_state == RESTARTER_STATE_DEGRADED))
4415                                 return (B_FALSE);
4416                 } else {
4417                         /*
4418                          * Skip all excluded dependents and decide whether
4419                          * to offline the service based on the restart_on
4420                          * on attribute.
4421                          */
4422                         if (is_depgrp_bypassed(vv))
4423                                 continue;
4424 
4425                         /*
4426                          * For dependency groups or service vertices, keep
4427                          * traversing to see if instances are running.
4428                          */
4429                         if (insubtree_dependents_down(vv) == B_FALSE)
4430                                 return (B_FALSE);
4431                 }
4432         }
4433 
4434         return (B_TRUE);
4435 }
4436 
4437 /*
4438  * Returns true only if none of this service's dependents are 'up' -- online,
4439  * degraded, or offline.
4440  */
4441 static int
4442 is_nonsubgraph_leaf(graph_vertex_t *v)
4443 {
4444         graph_vertex_t *vv;
4445         graph_edge_t *e;
4446 
4447         assert(MUTEX_HELD(&dgraph_lock));
4448 
4449         for (e = uu_list_first(v->gv_dependents);
4450             e != NULL;
4451             e = uu_list_next(v->gv_dependents, e)) {
4452 
4453                 vv = e->ge_vertex;
4454                 if (vv->gv_type == GVT_INST) {
4455                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4456                                 continue;
4457 
4458                         if (vv->gv_flags & GV_INSUBGRAPH)
4459                                 continue;
4460 
4461                         if (up_state(vv->gv_state))
4462                                 return (0);
4463                 } else {
4464                         /*
4465                          * For dependency group or service vertices, keep
4466                          * traversing to see if instances are running.
4467                          *
4468                          * We should skip exclude_all dependencies otherwise
4469                          * the vertex will never be considered as a leaf
4470                          * if the dependent is offline. The main reason for
4471                          * this is that disable_nonsubgraph_leaves() skips
4472                          * exclusion dependencies.
4473                          */
4474                         if (vv->gv_type == GVT_GROUP &&
4475                             vv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4476                                 continue;
4477 
4478                         if (!is_nonsubgraph_leaf(vv))
4479                                 return (0);
4480                 }
4481         }
4482 
4483         return (1);
4484 }
4485 
4486 /*
4487  * Disable v temporarily.  Attempt to do this by setting its enabled override
4488  * property in the repository.  If that fails, send a _DISABLE command.
4489  * Returns 0 on success and ECONNABORTED if the repository connection is
4490  * broken.
4491  */
4492 static int
4493 disable_service_temporarily(graph_vertex_t *v, scf_handle_t *h)
4494 {
4495         const char * const emsg = "Could not temporarily disable %s because "
4496             "%s.  Will stop service anyways.  Repository status for the "
4497             "service may be inaccurate.\n";
4498         const char * const emsg_cbroken =
4499             "the repository connection was broken";
4500 
4501         scf_instance_t *inst;
4502         int r;
4503 
4504         inst = scf_instance_create(h);
4505         if (inst == NULL) {
4506                 char buf[100];
4507 
4508                 (void) snprintf(buf, sizeof (buf),
4509                     "scf_instance_create() failed (%s)",
4510                     scf_strerror(scf_error()));
4511                 log_error(LOG_WARNING, emsg, v->gv_name, buf);
4512 
4513                 graph_enable_by_vertex(v, 0, 0);
4514                 return (0);
4515         }
4516 
4517         r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
4518             NULL, NULL, SCF_DECODE_FMRI_EXACT);
4519         if (r != 0) {
4520                 switch (scf_error()) {
4521                 case SCF_ERROR_CONNECTION_BROKEN:
4522                         log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4523                         graph_enable_by_vertex(v, 0, 0);
4524                         return (ECONNABORTED);
4525 
4526                 case SCF_ERROR_NOT_FOUND:
4527                         return (0);
4528 
4529                 case SCF_ERROR_HANDLE_MISMATCH:
4530                 case SCF_ERROR_INVALID_ARGUMENT:
4531                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4532                 case SCF_ERROR_NOT_BOUND:
4533                 default:
4534                         bad_error("scf_handle_decode_fmri",
4535                             scf_error());
4536                 }
4537         }
4538 
4539         r = libscf_set_enable_ovr(inst, 0);
4540         switch (r) {
4541         case 0:
4542                 scf_instance_destroy(inst);
4543                 return (0);
4544 
4545         case ECANCELED:
4546                 scf_instance_destroy(inst);
4547                 return (0);
4548 
4549         case ECONNABORTED:
4550                 log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4551                 graph_enable_by_vertex(v, 0, 0);
4552                 return (ECONNABORTED);
4553 
4554         case EPERM:
4555                 log_error(LOG_WARNING, emsg, v->gv_name,
4556                     "the repository denied permission");
4557                 graph_enable_by_vertex(v, 0, 0);
4558                 return (0);
4559 
4560         case EROFS:
4561                 log_error(LOG_WARNING, emsg, v->gv_name,
4562                     "the repository is read-only");
4563                 graph_enable_by_vertex(v, 0, 0);
4564                 return (0);
4565 
4566         default:
4567                 bad_error("libscf_set_enable_ovr", r);
4568                 /* NOTREACHED */
4569         }
4570 }
4571 
4572 /*
4573  * Of the transitive instance dependencies of v, offline those which are
4574  * in the subtree and which are leaves (i.e., have no dependents which are
4575  * "up").
4576  */
4577 void
4578 offline_subtree_leaves(graph_vertex_t *v, void *arg)
4579 {
4580         assert(MUTEX_HELD(&dgraph_lock));
4581 
4582         /* If v isn't an instance, recurse on its dependencies. */
4583         if (v->gv_type != GVT_INST) {
4584                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4585                 return;
4586         }
4587 
4588         /*
4589          * If v is not in the subtree, so should all of its dependencies,
4590          * so do nothing.
4591          */
4592         if ((v->gv_flags & GV_TOOFFLINE) == 0)
4593                 return;
4594 
4595         /* If v isn't a leaf because it's already down, recurse. */
4596         if (!up_state(v->gv_state)) {
4597                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4598                 return;
4599         }
4600 
4601         /* if v is a leaf, offline it or disable it if it's the last one */
4602         if (insubtree_dependents_down(v) == B_TRUE) {
4603                 if (v->gv_flags & GV_TODISABLE)
4604                         vertex_send_event(v,
4605                             RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
4606                 else
4607                         offline_vertex(v);
4608         }
4609 }
4610 
4611 void
4612 graph_offline_subtree_leaves(graph_vertex_t *v, void *h)
4613 {
4614         graph_walk_dependencies(v, offline_subtree_leaves, (void *)h);
4615 }
4616 
4617 
4618 /*
4619  * Of the transitive instance dependencies of v, disable those which are not
4620  * in the subgraph and which are leaves (i.e., have no dependents which are
4621  * "up").
4622  */
4623 static void
4624 disable_nonsubgraph_leaves(graph_vertex_t *v, void *arg)
4625 {
4626         assert(MUTEX_HELD(&dgraph_lock));
4627 
4628         /*
4629          * We must skip exclusion dependencies because they are allowed to
4630          * complete dependency cycles.  This is correct because A's exclusion
4631          * dependency on B doesn't bear on the order in which they should be
4632          * stopped.  Indeed, the exclusion dependency should guarantee that
4633          * they are never online at the same time.
4634          */
4635         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4636                 return;
4637 
4638         /* If v isn't an instance, recurse on its dependencies. */
4639         if (v->gv_type != GVT_INST)
4640                 goto recurse;
4641 
4642         if ((v->gv_flags & GV_CONFIGURED) == 0)
4643                 /*
4644                  * Unconfigured instances should have no dependencies, but in
4645                  * case they ever get them,
4646                  */
4647                 goto recurse;
4648 
4649         /*
4650          * If v is in the subgraph, so should all of its dependencies, so do
4651          * nothing.
4652          */
4653         if (v->gv_flags & GV_INSUBGRAPH)
4654                 return;
4655 
4656         /* If v isn't a leaf because it's already down, recurse. */
4657         if (!up_state(v->gv_state))
4658                 goto recurse;
4659 
4660         /* If v is disabled but not down yet, be patient. */
4661         if ((v->gv_flags & GV_ENABLED) == 0)
4662                 return;
4663 
4664         /* If v is a leaf, disable it. */
4665         if (is_nonsubgraph_leaf(v))
4666                 (void) disable_service_temporarily(v, (scf_handle_t *)arg);
4667 
4668         return;
4669 
4670 recurse:
4671         graph_walk_dependencies(v, disable_nonsubgraph_leaves, arg);
4672 }
4673 
4674 static int
4675 stn_restarter_state(restarter_instance_state_t rstate)
4676 {
4677         static const struct statemap {
4678                 restarter_instance_state_t restarter_state;
4679                 int scf_state;
4680         } map[] = {
4681                 { RESTARTER_STATE_UNINIT, SCF_STATE_UNINIT },
4682                 { RESTARTER_STATE_MAINT, SCF_STATE_MAINT },
4683                 { RESTARTER_STATE_OFFLINE, SCF_STATE_OFFLINE },
4684                 { RESTARTER_STATE_DISABLED, SCF_STATE_DISABLED },
4685                 { RESTARTER_STATE_ONLINE, SCF_STATE_ONLINE },
4686                 { RESTARTER_STATE_DEGRADED, SCF_STATE_DEGRADED }
4687         };
4688 
4689         int i;
4690 
4691         for (i = 0; i < sizeof (map) / sizeof (map[0]); i++) {
4692                 if (rstate == map[i].restarter_state)
4693                         return (map[i].scf_state);
4694         }
4695 
4696         return (-1);
4697 }
4698 
4699 /*
4700  * State transition counters
4701  * Not incremented atomically - indicative only
4702  */
4703 static uint64_t stev_ct_maint;
4704 static uint64_t stev_ct_hwerr;
4705 static uint64_t stev_ct_service;
4706 static uint64_t stev_ct_global;
4707 static uint64_t stev_ct_noprefs;
4708 static uint64_t stev_ct_from_uninit;
4709 static uint64_t stev_ct_bad_state;
4710 static uint64_t stev_ct_ovr_prefs;
4711 
4712 static void
4713 dgraph_state_transition_notify(graph_vertex_t *v,
4714     restarter_instance_state_t old_state, restarter_str_t reason)
4715 {
4716         restarter_instance_state_t new_state = v->gv_state;
4717         int stn_transition, maint;
4718         int from, to;
4719         nvlist_t *attr;
4720         fmev_pri_t pri = FMEV_LOPRI;
4721         int raise = 0;
4722 
4723         if ((from = stn_restarter_state(old_state)) == -1 ||
4724             (to = stn_restarter_state(new_state)) == -1) {
4725                 stev_ct_bad_state++;
4726                 return;
4727         }
4728 
4729         stn_transition = from << 16 | to;
4730 
4731         maint = (to == SCF_STATE_MAINT || from == SCF_STATE_MAINT);
4732 
4733         if (maint) {
4734                 /*
4735                  * All transitions to/from maintenance state must raise
4736                  * an event.
4737                  */
4738                 raise++;
4739                 pri = FMEV_HIPRI;
4740                 stev_ct_maint++;
4741         } else if (reason == restarter_str_ct_ev_hwerr) {
4742                 /*
4743                  * All transitions caused by hardware fault must raise
4744                  * an event
4745                  */
4746                 raise++;
4747                 pri = FMEV_HIPRI;
4748                 stev_ct_hwerr++;
4749         } else if (stn_transition & v->gv_stn_tset) {
4750                 /*
4751                  * Specifically enabled event.
4752                  */
4753                 raise++;
4754                 stev_ct_service++;
4755         } else if (from == SCF_STATE_UNINIT) {
4756                 /*
4757                  * Only raise these if specifically selected above.
4758                  */
4759                 stev_ct_from_uninit++;
4760         } else if (stn_transition & stn_global &&
4761             (IS_ENABLED(v) == 1 || to == SCF_STATE_DISABLED)) {
4762                 raise++;
4763                 stev_ct_global++;
4764         } else {
4765                 stev_ct_noprefs++;
4766         }
4767 
4768         if (info_events_all) {
4769                 stev_ct_ovr_prefs++;
4770                 raise++;
4771         }
4772         if (!raise)
4773                 return;
4774 
4775         if (nvlist_alloc(&attr, NV_UNIQUE_NAME, 0) != 0 ||
4776             nvlist_add_string(attr, "fmri", v->gv_name) != 0 ||
4777             nvlist_add_uint32(attr, "reason-version",
4778             restarter_str_version()) || nvlist_add_string(attr, "reason-short",
4779             restarter_get_str_short(reason)) != 0 ||
4780             nvlist_add_string(attr, "reason-long",
4781             restarter_get_str_long(reason)) != 0 ||
4782             nvlist_add_int32(attr, "transition", stn_transition) != 0) {
4783                 log_framework(LOG_WARNING,
4784                     "FMEV: %s could not create nvlist for transition "
4785                     "event: %s\n", v->gv_name, strerror(errno));
4786                 nvlist_free(attr);
4787                 return;
4788         }
4789 
4790         if (fmev_rspublish_nvl(FMEV_RULESET_SMF, "state-transition",
4791             instance_state_str[new_state], pri, attr) != FMEV_SUCCESS) {
4792                 log_framework(LOG_DEBUG,
4793                     "FMEV: %s failed to publish transition event: %s\n",
4794                     v->gv_name, fmev_strerror(fmev_errno));
4795                 nvlist_free(attr);
4796         }
4797 }
4798 
4799 /*
4800  * Find the vertex for inst_name.  If it doesn't exist, return ENOENT.
4801  * Otherwise set its state to state.  If the instance has entered a state
4802  * which requires automatic action, take it (Uninitialized: do
4803  * dgraph_refresh_instance() without the snapshot update.  Disabled: if the
4804  * instance should be enabled, send _ENABLE.  Offline: if the instance should
4805  * be disabled, send _DISABLE, and if its dependencies are satisfied, send
4806  * _START.  Online, Degraded: if the instance wasn't running, update its start
4807  * snapshot.  Maintenance: no action.)
4808  *
4809  * Also fails with ECONNABORTED, or EINVAL if state is invalid.
4810  */
4811 static int
4812 dgraph_set_instance_state(scf_handle_t *h, const char *inst_name,
4813     protocol_states_t *states)
4814 {
4815         graph_vertex_t *v;
4816         int err = 0;
4817         restarter_instance_state_t old_state;
4818         restarter_instance_state_t state = states->ps_state;
4819         restarter_error_t serr = states->ps_err;
4820 
4821         MUTEX_LOCK(&dgraph_lock);
4822 
4823         v = vertex_get_by_name(inst_name);
4824         if (v == NULL) {
4825                 MUTEX_UNLOCK(&dgraph_lock);
4826                 return (ENOENT);
4827         }
4828 
4829         assert(v->gv_type == GVT_INST);
4830 
4831         switch (state) {
4832         case RESTARTER_STATE_UNINIT:
4833         case RESTARTER_STATE_DISABLED:
4834         case RESTARTER_STATE_OFFLINE:
4835         case RESTARTER_STATE_ONLINE:
4836         case RESTARTER_STATE_DEGRADED:
4837         case RESTARTER_STATE_MAINT:
4838                 break;
4839 
4840         default:
4841                 MUTEX_UNLOCK(&dgraph_lock);
4842                 return (EINVAL);
4843         }
4844 
4845         log_framework(LOG_DEBUG, "Graph noting %s %s -> %s.\n", v->gv_name,
4846             instance_state_str[v->gv_state], instance_state_str[state]);
4847 
4848         old_state = v->gv_state;
4849         v->gv_state = state;
4850 
4851         v->gv_reason = states->ps_reason;
4852         err = gt_transition(h, v, serr, old_state);
4853         if (err == 0 && v->gv_state != old_state) {
4854                 dgraph_state_transition_notify(v, old_state, states->ps_reason);
4855         }
4856 
4857         MUTEX_UNLOCK(&dgraph_lock);
4858         return (err);
4859 }
4860 
4861 /*
4862  * Handle state changes during milestone shutdown.  See
4863  * dgraph_set_milestone().  If the repository connection is broken,
4864  * ECONNABORTED will be returned, though a _DISABLE command will be sent for
4865  * the vertex anyway.
4866  */
4867 int
4868 vertex_subgraph_dependencies_shutdown(scf_handle_t *h, graph_vertex_t *v,
4869     restarter_instance_state_t old_state)
4870 {
4871         int was_up, now_up;
4872         int ret = 0;
4873 
4874         assert(v->gv_type == GVT_INST);
4875 
4876         /* Don't care if we're not going to a milestone. */
4877         if (milestone == NULL)
4878                 return (0);
4879 
4880         /* Don't care if we already finished coming down. */
4881         if (non_subgraph_svcs == 0)
4882                 return (0);
4883 
4884         /* Don't care if the service is in the subgraph. */
4885         if (v->gv_flags & GV_INSUBGRAPH)
4886                 return (0);
4887 
4888         /*
4889          * Update non_subgraph_svcs.  It is the number of non-subgraph
4890          * services which are in online, degraded, or offline.
4891          */
4892 
4893         was_up = up_state(old_state);
4894         now_up = up_state(v->gv_state);
4895 
4896         if (!was_up && now_up) {
4897                 ++non_subgraph_svcs;
4898         } else if (was_up && !now_up) {
4899                 --non_subgraph_svcs;
4900 
4901                 if (non_subgraph_svcs == 0) {
4902                         if (halting != -1) {
4903                                 do_uadmin();
4904                         } else if (go_single_user_mode || go_to_level1) {
4905                                 (void) startd_thread_create(single_user_thread,
4906                                     NULL);
4907                         }
4908                         return (0);
4909                 }
4910         }
4911 
4912         /* If this service is a leaf, it should be disabled. */
4913         if ((v->gv_flags & GV_ENABLED) && is_nonsubgraph_leaf(v)) {
4914                 int r;
4915 
4916                 r = disable_service_temporarily(v, h);
4917                 switch (r) {
4918                 case 0:
4919                         break;
4920 
4921                 case ECONNABORTED:
4922                         ret = ECONNABORTED;
4923                         break;
4924 
4925                 default:
4926                         bad_error("disable_service_temporarily", r);
4927                 }
4928         }
4929 
4930         /*
4931          * If the service just came down, propagate the disable to the newly
4932          * exposed leaves.
4933          */
4934         if (was_up && !now_up)
4935                 graph_walk_dependencies(v, disable_nonsubgraph_leaves,
4936                     (void *)h);
4937 
4938         return (ret);
4939 }
4940 
4941 /*
4942  * Decide whether to start up an sulogin thread after a service is
4943  * finished changing state.  Only need to do the full can_come_up()
4944  * evaluation if an instance is changing state, we're not halfway through
4945  * loading the thread, and we aren't shutting down or going to the single
4946  * user milestone.
4947  */
4948 void
4949 graph_transition_sulogin(restarter_instance_state_t state,
4950     restarter_instance_state_t old_state)
4951 {
4952         assert(MUTEX_HELD(&dgraph_lock));
4953 
4954         if (state != old_state && st->st_load_complete &&
4955             !go_single_user_mode && !go_to_level1 &&
4956             halting == -1) {
4957                 if (!sulogin_thread_running && !can_come_up()) {
4958                         (void) startd_thread_create(sulogin_thread, NULL);
4959                         sulogin_thread_running = B_TRUE;
4960                 }
4961         }
4962 }
4963 
4964 /*
4965  * Propagate a start, stop event, or a satisfiability event.
4966  *
4967  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4968  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4969  * full dependent graph to check for newly satisfied nodes.  This is
4970  * necessary for cases when non-direct dependents may be effected but direct
4971  * dependents may not (e.g. for optional_all evaluations, see the
4972  * propagate_satbility() comments).
4973  *
4974  * PROPAGATE_SAT should be used whenever a non-running service moves into
4975  * a state which can satisfy optional dependencies, like disabled or
4976  * maintenance.
4977  */
4978 void
4979 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4980     restarter_error_t rerr)
4981 {
4982         if (type == PROPAGATE_STOP) {
4983                 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4984         } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4985                 graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
4986 
4987                 if (type == PROPAGATE_SAT)
4988                         propagate_satbility(v);
4989         } else {
4990 #ifndef NDEBUG
4991                 uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4992                     __LINE__, type);
4993 #endif
4994                 abort();
4995         }
4996 }
4997 
4998 /*
4999  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
5000  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
5001  * all property group dependencies, and the dependency on the restarter,
5002  * disposing of vertices as appropriate.  If other vertices depend on this
5003  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
5004  * returns 0.
5005  */
5006 static int
5007 dgraph_remove_instance(const char *fmri, scf_handle_t *h)
5008 {
5009         graph_vertex_t *v;
5010         graph_edge_t *e;
5011         uu_list_t *old_deps;
5012         int err;
5013 
5014         log_framework(LOG_DEBUG, "Graph engine: Removing %s.\n", fmri);
5015 
5016         MUTEX_LOCK(&dgraph_lock);
5017 
5018         v = vertex_get_by_name(fmri);
5019         if (v == NULL) {
5020                 MUTEX_UNLOCK(&dgraph_lock);
5021                 return (0);
5022         }
5023 
5024         /* Send restarter delete event. */
5025         if (v->gv_flags & GV_CONFIGURED)
5026                 graph_unset_restarter(v);
5027 
5028         if (milestone > MILESTONE_NONE) {
5029                 /*
5030                  * Make a list of v's current dependencies so we can
5031                  * reevaluate their GV_INSUBGRAPH flags after the dependencies
5032                  * are removed.
5033                  */
5034                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
5035 
5036                 err = uu_list_walk(v->gv_dependencies,
5037                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
5038                 assert(err == 0);
5039         }
5040 
5041         delete_instance_dependencies(v, B_TRUE);
5042 
5043         /*
5044          * Deleting an instance can both satisfy and unsatisfy dependencies,
5045          * depending on their type.  First propagate the stop as a RERR_RESTART
5046          * event -- deletion isn't a fault, just a normal stop.  This gives
5047          * dependent services the chance to do a clean shutdown.  Then, mark
5048          * the service as unconfigured and propagate the start event for the
5049          * optional_all dependencies that might have become satisfied.
5050          */
5051         graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
5052 
5053         v->gv_flags &= ~GV_CONFIGURED;
5054         v->gv_flags &= ~GV_DEATHROW;
5055 
5056         graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
5057         propagate_satbility(v);
5058 
5059         /*
5060          * If there are no (non-service) dependents, the vertex can be
5061          * completely removed.
5062          */
5063         if (v != milestone && v->gv_refs == 0 &&
5064             uu_list_numnodes(v->gv_dependents) == 1)
5065                 remove_inst_vertex(v);
5066 
5067         if (milestone > MILESTONE_NONE) {
5068                 void *cookie = NULL;
5069 
5070                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5071                         v = e->ge_vertex;
5072 
5073                         if (vertex_unref(v) == VERTEX_INUSE)
5074                                 while (eval_subgraph(v, h) == ECONNABORTED)
5075                                         libscf_handle_rebind(h);
5076 
5077                         startd_free(e, sizeof (*e));
5078                 }
5079 
5080                 uu_list_destroy(old_deps);
5081         }
5082 
5083         MUTEX_UNLOCK(&dgraph_lock);
5084 
5085         return (0);
5086 }
5087 
5088 /*
5089  * Return the eventual (maybe current) milestone in the form of a
5090  * legacy runlevel.
5091  */
5092 static char
5093 target_milestone_as_runlevel()
5094 {
5095         assert(MUTEX_HELD(&dgraph_lock));
5096 
5097         if (milestone == NULL)
5098                 return ('3');
5099         else if (milestone == MILESTONE_NONE)
5100                 return ('0');
5101 
5102         if (strcmp(milestone->gv_name, multi_user_fmri) == 0)
5103                 return ('2');
5104         else if (strcmp(milestone->gv_name, single_user_fmri) == 0)
5105                 return ('S');
5106         else if (strcmp(milestone->gv_name, multi_user_svr_fmri) == 0)
5107                 return ('3');
5108 
5109 #ifndef NDEBUG
5110         (void) fprintf(stderr, "%s:%d: Unknown milestone name \"%s\".\n",
5111             __FILE__, __LINE__, milestone->gv_name);
5112 #endif
5113         abort();
5114         /* NOTREACHED */
5115 }
5116 
5117 static struct {
5118         char    rl;
5119         int     sig;
5120 } init_sigs[] = {
5121         { 'S', SIGBUS },
5122         { '0', SIGINT },
5123         { '1', SIGQUIT },
5124         { '2', SIGILL },
5125         { '3', SIGTRAP },
5126         { '4', SIGIOT },
5127         { '5', SIGEMT },
5128         { '6', SIGFPE },
5129         { 0, 0 }
5130 };
5131 
5132 static void
5133 signal_init(char rl)
5134 {
5135         pid_t init_pid;
5136         int i;
5137 
5138         assert(MUTEX_HELD(&dgraph_lock));
5139 
5140         if (zone_getattr(getzoneid(), ZONE_ATTR_INITPID, &init_pid,
5141             sizeof (init_pid)) != sizeof (init_pid)) {
5142                 log_error(LOG_NOTICE, "Could not get pid to signal init.\n");
5143                 return;
5144         }
5145 
5146         for (i = 0; init_sigs[i].rl != 0; ++i)
5147                 if (init_sigs[i].rl == rl)
5148                         break;
5149 
5150         if (init_sigs[i].rl != 0) {
5151                 if (kill(init_pid, init_sigs[i].sig) != 0) {
5152                         switch (errno) {
5153                         case EPERM:
5154                         case ESRCH:
5155                                 log_error(LOG_NOTICE, "Could not signal init: "
5156                                     "%s.\n", strerror(errno));
5157                                 break;
5158 
5159                         case EINVAL:
5160                         default:
5161                                 bad_error("kill", errno);
5162                         }
5163                 }
5164         }
5165 }
5166 
5167 /*
5168  * This is called when one of the major milestones changes state, or when
5169  * init is signalled and tells us it was told to change runlevel.  We wait
5170  * to reach the milestone because this allows /etc/inittab entries to retain
5171  * some boot ordering: historically, entries could place themselves before/after
5172  * the running of /sbin/rcX scripts but we can no longer make the
5173  * distinction because the /sbin/rcX scripts no longer exist as punctuation
5174  * marks in /etc/inittab.
5175  *
5176  * Also, we only trigger an update when we reach the eventual target
5177  * milestone: without this, an /etc/inittab entry marked only for
5178  * runlevel 2 would be executed for runlevel 3, which is not how
5179  * /etc/inittab entries work.
5180  *
5181  * If we're single user coming online, then we set utmpx to the target
5182  * runlevel so that legacy scripts can work as expected.
5183  */
5184 static void
5185 graph_runlevel_changed(char rl, int online)
5186 {
5187         char trl;
5188 
5189         assert(MUTEX_HELD(&dgraph_lock));
5190 
5191         trl = target_milestone_as_runlevel();
5192 
5193         if (online) {
5194                 if (rl == trl) {
5195                         current_runlevel = trl;
5196                         signal_init(trl);
5197                 } else if (rl == 'S') {
5198                         /*
5199                          * At boot, set the entry early for the benefit of the
5200                          * legacy init scripts.
5201                          */
5202                         utmpx_set_runlevel(trl, 'S', B_FALSE);
5203                 }
5204         } else {
5205                 if (rl == '3' && trl == '2') {
5206                         current_runlevel = trl;
5207                         signal_init(trl);
5208                 } else if (rl == '2' && trl == 'S') {
5209                         current_runlevel = trl;
5210                         signal_init(trl);
5211                 }
5212         }
5213 }
5214 
5215 /*
5216  * Move to a backwards-compatible runlevel by executing the appropriate
5217  * /etc/rc?.d/K* scripts and/or setting the milestone.
5218  *
5219  * Returns
5220  *   0 - success
5221  *   ECONNRESET - success, but handle was reset
5222  *   ECONNABORTED - repository connection broken
5223  *   ECANCELED - pg was deleted
5224  */
5225 static int
5226 dgraph_set_runlevel(scf_propertygroup_t *pg, scf_property_t *prop)
5227 {
5228         char rl;
5229         scf_handle_t *h;
5230         int r;
5231         const char *ms = NULL;  /* what to commit as options/milestone */
5232         boolean_t rebound = B_FALSE;
5233         int mark_rl = 0;
5234 
5235         const char * const stop = "stop";
5236 
5237         r = libscf_extract_runlevel(prop, &rl);
5238         switch (r) {
5239         case 0:
5240                 break;
5241 
5242         case ECONNABORTED:
5243         case ECANCELED:
5244                 return (r);
5245 
5246         case EINVAL:
5247         case ENOENT:
5248                 log_error(LOG_WARNING, "runlevel property is misconfigured; "
5249                     "ignoring.\n");
5250                 /* delete the bad property */
5251                 goto nolock_out;
5252 
5253         default:
5254                 bad_error("libscf_extract_runlevel", r);
5255         }
5256 
5257         switch (rl) {
5258         case 's':
5259                 rl = 'S';
5260                 /* FALLTHROUGH */
5261 
5262         case 'S':
5263         case '2':
5264         case '3':
5265                 /*
5266                  * These cases cause a milestone change, so
5267                  * graph_runlevel_changed() will eventually deal with
5268                  * signalling init.
5269                  */
5270                 break;
5271 
5272         case '0':
5273         case '1':
5274         case '4':
5275         case '5':
5276         case '6':
5277                 mark_rl = 1;
5278                 break;
5279 
5280         default:
5281                 log_framework(LOG_NOTICE, "Unknown runlevel '%c'.\n", rl);
5282                 ms = NULL;
5283                 goto nolock_out;
5284         }
5285 
5286         h = scf_pg_handle(pg);
5287 
5288         MUTEX_LOCK(&dgraph_lock);
5289 
5290         /*
5291          * Since this triggers no milestone changes, force it by hand.
5292          */
5293         if (current_runlevel == '4' && rl == '3')
5294                 mark_rl = 1;
5295 
5296         /*
5297          * 1. If we are here after an "init X":
5298          *
5299          * init X
5300          *      init/lscf_set_runlevel()
5301          *              process_pg_event()
5302          *              dgraph_set_runlevel()
5303          *
5304          * then we haven't passed through graph_runlevel_changed() yet,
5305          * therefore 'current_runlevel' has not changed for sure but 'rl' has.
5306          * In consequence, if 'rl' is lower than 'current_runlevel', we change
5307          * the system runlevel and execute the appropriate /etc/rc?.d/K* scripts
5308          * past this test.
5309          *
5310          * 2. On the other hand, if we are here after a "svcadm milestone":
5311          *
5312          * svcadm milestone X
5313          *      dgraph_set_milestone()
5314          *              handle_graph_update_event()
5315          *              dgraph_set_instance_state()
5316          *              graph_post_X_[online|offline]()
5317          *              graph_runlevel_changed()
5318          *              signal_init()
5319          *                      init/lscf_set_runlevel()
5320          *                              process_pg_event()
5321          *                              dgraph_set_runlevel()
5322          *
5323          * then we already passed through graph_runlevel_changed() (by the way
5324          * of dgraph_set_milestone()) and 'current_runlevel' may have changed
5325          * and already be equal to 'rl' so we are going to return immediately
5326          * from dgraph_set_runlevel() without changing the system runlevel and
5327          * without executing the /etc/rc?.d/K* scripts.
5328          */
5329         if (rl == current_runlevel) {
5330                 ms = NULL;
5331                 goto out;
5332         }
5333 
5334         log_framework(LOG_DEBUG, "Changing to runlevel '%c'.\n", rl);
5335 
5336         /*
5337          * Make sure stop rc scripts see the new settings via who -r.
5338          */
5339         utmpx_set_runlevel(rl, current_runlevel, B_TRUE);
5340 
5341         /*
5342          * Some run levels don't have a direct correspondence to any
5343          * milestones, so we have to signal init directly.
5344          */
5345         if (mark_rl) {
5346                 current_runlevel = rl;
5347                 signal_init(rl);
5348         }
5349 
5350         switch (rl) {
5351         case 'S':
5352                 uu_warn("The system is coming down for administration.  "
5353                     "Please wait.\n");
5354                 fork_rc_script(rl, stop, B_FALSE);
5355                 ms = single_user_fmri;
5356                 go_single_user_mode = B_TRUE;
5357                 break;
5358 
5359         case '0':
5360                 halting_time = time(NULL);
5361                 fork_rc_script(rl, stop, B_TRUE);
5362                 halting = AD_HALT;
5363                 goto uadmin;
5364 
5365         case '5':
5366                 halting_time = time(NULL);
5367                 fork_rc_script(rl, stop, B_TRUE);
5368                 halting = AD_POWEROFF;
5369                 goto uadmin;
5370 
5371         case '6':
5372                 halting_time = time(NULL);
5373                 fork_rc_script(rl, stop, B_TRUE);
5374                 if (scf_is_fastboot_default() && getzoneid() == GLOBAL_ZONEID)
5375                         halting = AD_FASTREBOOT;
5376                 else
5377                         halting = AD_BOOT;
5378 
5379 uadmin:
5380                 uu_warn("The system is coming down.  Please wait.\n");
5381                 ms = "none";
5382 
5383                 /*
5384                  * We can't wait until all services are offline since this
5385                  * thread is responsible for taking them offline.  Instead we
5386                  * set halting to the second argument for uadmin() and call
5387                  * do_uadmin() from dgraph_set_instance_state() when
5388                  * appropriate.
5389                  */
5390                 break;
5391 
5392         case '1':
5393                 if (current_runlevel != 'S') {
5394                         uu_warn("Changing to state 1.\n");
5395                         fork_rc_script(rl, stop, B_FALSE);
5396                 } else {
5397                         uu_warn("The system is coming up for administration.  "
5398                             "Please wait.\n");
5399                 }
5400                 ms = single_user_fmri;
5401                 go_to_level1 = B_TRUE;
5402                 break;
5403 
5404         case '2':
5405                 if (current_runlevel == '3' || current_runlevel == '4')
5406                         fork_rc_script(rl, stop, B_FALSE);
5407                 ms = multi_user_fmri;
5408                 break;
5409 
5410         case '3':
5411         case '4':
5412                 ms = "all";
5413                 break;
5414 
5415         default:
5416 #ifndef NDEBUG
5417                 (void) fprintf(stderr, "%s:%d: Uncaught case %d ('%c').\n",
5418                     __FILE__, __LINE__, rl, rl);
5419 #endif
5420                 abort();
5421         }
5422 
5423 out:
5424         MUTEX_UNLOCK(&dgraph_lock);
5425 
5426 nolock_out:
5427         switch (r = libscf_clear_runlevel(pg, ms)) {
5428         case 0:
5429                 break;
5430 
5431         case ECONNABORTED:
5432                 libscf_handle_rebind(h);
5433                 rebound = B_TRUE;
5434                 goto nolock_out;
5435 
5436         case ECANCELED:
5437                 break;
5438 
5439         case EPERM:
5440         case EACCES:
5441         case EROFS:
5442                 log_error(LOG_NOTICE, "Could not delete \"%s/%s\" property: "
5443                     "%s.\n", SCF_PG_OPTIONS, "runlevel", strerror(r));
5444                 break;
5445 
5446         default:
5447                 bad_error("libscf_clear_runlevel", r);
5448         }
5449 
5450         return (rebound ? ECONNRESET : 0);
5451 }
5452 
5453 /*
5454  * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5455  * to the instances that are supposed to go offline during an
5456  * administrative disable operation.
5457  */
5458 static int
5459 mark_subtree(graph_edge_t *e, void *arg)
5460 {
5461         graph_vertex_t *v;
5462         int r;
5463 
5464         v = e->ge_vertex;
5465 
5466         /* If it's already in the subgraph, skip. */
5467         if (v->gv_flags & GV_TOOFFLINE)
5468                 return (UU_WALK_NEXT);
5469 
5470         switch (v->gv_type) {
5471         case GVT_INST:
5472                 /* If the instance is already disabled, skip it. */
5473                 if (!(v->gv_flags & GV_ENABLED))
5474                         return (UU_WALK_NEXT);
5475 
5476                 v->gv_flags |= GV_TOOFFLINE;
5477                 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5478                 break;
5479         case GVT_GROUP:
5480                 /*
5481                  * Skip all excluded dependents and decide whether to offline
5482                  * the service based on the restart_on attribute.
5483                  */
5484                 if (is_depgrp_bypassed(v))
5485                         return (UU_WALK_NEXT);
5486                 break;
5487         }
5488 
5489         r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5490             0);
5491         assert(r == 0);
5492         return (UU_WALK_NEXT);
5493 }
5494 
5495 static int
5496 mark_subgraph(graph_edge_t *e, void *arg)
5497 {
5498         graph_vertex_t *v;
5499         int r;
5500         int optional = (int)arg;
5501 
5502         v = e->ge_vertex;
5503 
5504         /* If it's already in the subgraph, skip. */
5505         if (v->gv_flags & GV_INSUBGRAPH)
5506                 return (UU_WALK_NEXT);
5507 
5508         /*
5509          * Keep track if walk has entered an optional dependency group
5510          */
5511         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_OPTIONAL_ALL) {
5512                 optional = 1;
5513         }
5514         /*
5515          * Quit if we are in an optional dependency group and the instance
5516          * is disabled
5517          */
5518         if (optional && (v->gv_type == GVT_INST) &&
5519             (!(v->gv_flags & GV_ENBLD_NOOVR)))
5520                 return (UU_WALK_NEXT);
5521 
5522         v->gv_flags |= GV_INSUBGRAPH;
5523 
5524         /* Skip all excluded dependencies. */
5525         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
5526                 return (UU_WALK_NEXT);
5527 
5528         r = uu_list_walk(v->gv_dependencies, (uu_walk_fn_t *)mark_subgraph,
5529             (void *)optional, 0);
5530         assert(r == 0);
5531         return (UU_WALK_NEXT);
5532 }
5533 
5534 /*
5535  * Bring down all services which are not dependencies of fmri.  The
5536  * dependencies of fmri (direct & indirect) will constitute the "subgraph",
5537  * and will have the GV_INSUBGRAPH flag set.  The rest must be brought down,
5538  * which means the state is "disabled", "maintenance", or "uninitialized".  We
5539  * could consider "offline" to be down, and refrain from sending start
5540  * commands for such services, but that's not strictly necessary, so we'll
5541  * decline to intrude on the state machine.  It would probably confuse users
5542  * anyway.
5543  *
5544  * The services should be brought down in reverse-dependency order, so we
5545  * can't do it all at once here.  We initiate by override-disabling the leaves
5546  * of the dependency tree -- those services which are up but have no
5547  * dependents which are up.  When they come down,
5548  * vertex_subgraph_dependencies_shutdown() will override-disable the newly
5549  * exposed leaves.  Perseverance will ensure completion.
5550  *
5551  * Sometimes we need to take action when the transition is complete, like
5552  * start sulogin or halt the system.  To tell when we're done, we initialize
5553  * non_subgraph_svcs here to be the number of services which need to come
5554  * down.  As each does, we decrement the counter.  When it hits zero, we take
5555  * the appropriate action.  See vertex_subgraph_dependencies_shutdown().
5556  *
5557  * In case we're coming up, we also remove any enable-overrides for the
5558  * services which are dependencies of fmri.
5559  *
5560  * If norepository is true, the function will not change the repository.
5561  *
5562  * The decision to change the system run level in accordance with the milestone
5563  * is taken in dgraph_set_runlevel().
5564  *
5565  * Returns
5566  *   0 - success
5567  *   ECONNRESET - success, but handle was rebound
5568  *   EINVAL - fmri is invalid (error is logged)
5569  *   EALREADY - the milestone is already set to fmri
5570  *   ENOENT - a configured vertex does not exist for fmri (an error is logged)
5571  */
5572 static int
5573 dgraph_set_milestone(const char *fmri, scf_handle_t *h, boolean_t norepository)
5574 {
5575         const char *cfmri, *fs;
5576         graph_vertex_t *nm, *v;
5577         int ret = 0, r;
5578         scf_instance_t *inst;
5579         boolean_t isall, isnone, rebound = B_FALSE;
5580 
5581         /* Validate fmri */
5582         isall = (strcmp(fmri, "all") == 0);
5583         isnone = (strcmp(fmri, "none") == 0);
5584 
5585         if (!isall && !isnone) {
5586                 if (fmri_canonify(fmri, (char **)&cfmri, B_FALSE) == EINVAL)
5587                         goto reject;
5588 
5589                 if (strcmp(cfmri, single_user_fmri) != 0 &&
5590                     strcmp(cfmri, multi_user_fmri) != 0 &&
5591                     strcmp(cfmri, multi_user_svr_fmri) != 0) {
5592                         startd_free((void *)cfmri, max_scf_fmri_size);
5593 reject:
5594                         log_framework(LOG_WARNING,
5595                             "Rejecting request for invalid milestone \"%s\".\n",
5596                             fmri);
5597                         return (EINVAL);
5598                 }
5599         }
5600 
5601         inst = safe_scf_instance_create(h);
5602 
5603         MUTEX_LOCK(&dgraph_lock);
5604 
5605         if (milestone == NULL) {
5606                 if (isall) {
5607                         log_framework(LOG_DEBUG,
5608                             "Milestone already set to all.\n");
5609                         ret = EALREADY;
5610                         goto out;
5611                 }
5612         } else if (milestone == MILESTONE_NONE) {
5613                 if (isnone) {
5614                         log_framework(LOG_DEBUG,
5615                             "Milestone already set to none.\n");
5616                         ret = EALREADY;
5617                         goto out;
5618                 }
5619         } else {
5620                 if (!isall && !isnone &&
5621                     strcmp(cfmri, milestone->gv_name) == 0) {
5622                         log_framework(LOG_DEBUG,
5623                             "Milestone already set to %s.\n", cfmri);
5624                         ret = EALREADY;
5625                         goto out;
5626                 }
5627         }
5628 
5629         if (!isall && !isnone) {
5630                 nm = vertex_get_by_name(cfmri);
5631                 if (nm == NULL || !(nm->gv_flags & GV_CONFIGURED)) {
5632                         log_framework(LOG_WARNING, "Cannot set milestone to %s "
5633                             "because no such service exists.\n", cfmri);
5634                         ret = ENOENT;
5635                         goto out;
5636                 }
5637         }
5638 
5639         log_framework(LOG_DEBUG, "Changing milestone to %s.\n", fmri);
5640 
5641         /*
5642          * Set milestone, removing the old one if this was the last reference.
5643          */
5644         if (milestone > MILESTONE_NONE)
5645                 (void) vertex_unref(milestone);
5646 
5647         if (isall)
5648                 milestone = NULL;
5649         else if (isnone)
5650                 milestone = MILESTONE_NONE;
5651         else {
5652                 milestone = nm;
5653                 /* milestone should count as a reference */
5654                 vertex_ref(milestone);
5655         }
5656 
5657         /* Clear all GV_INSUBGRAPH bits. */
5658         for (v = uu_list_first(dgraph); v != NULL; v = uu_list_next(dgraph, v))
5659                 v->gv_flags &= ~GV_INSUBGRAPH;
5660 
5661         if (!isall && !isnone) {
5662                 /* Set GV_INSUBGRAPH for milestone & descendents. */
5663                 milestone->gv_flags |= GV_INSUBGRAPH;
5664 
5665                 r = uu_list_walk(milestone->gv_dependencies,
5666                     (uu_walk_fn_t *)mark_subgraph, NULL, 0);
5667                 assert(r == 0);
5668         }
5669 
5670         /* Un-override services in the subgraph & override-disable the rest. */
5671         if (norepository)
5672                 goto out;
5673 
5674         non_subgraph_svcs = 0;
5675         for (v = uu_list_first(dgraph);
5676             v != NULL;
5677             v = uu_list_next(dgraph, v)) {
5678                 if (v->gv_type != GVT_INST ||
5679                     (v->gv_flags & GV_CONFIGURED) == 0)
5680                         continue;
5681 
5682 again:
5683                 r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
5684                     NULL, NULL, SCF_DECODE_FMRI_EXACT);
5685                 if (r != 0) {
5686                         switch (scf_error()) {
5687                         case SCF_ERROR_CONNECTION_BROKEN:
5688                         default:
5689                                 libscf_handle_rebind(h);
5690                                 rebound = B_TRUE;
5691                                 goto again;
5692 
5693                         case SCF_ERROR_NOT_FOUND:
5694                                 continue;
5695 
5696                         case SCF_ERROR_HANDLE_MISMATCH:
5697                         case SCF_ERROR_INVALID_ARGUMENT:
5698                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5699                         case SCF_ERROR_NOT_BOUND:
5700                                 bad_error("scf_handle_decode_fmri",
5701                                     scf_error());
5702                         }
5703                 }
5704 
5705                 if (isall || (v->gv_flags & GV_INSUBGRAPH)) {
5706                         r = libscf_delete_enable_ovr(inst);
5707                         fs = "libscf_delete_enable_ovr";
5708                 } else {
5709                         assert(isnone || (v->gv_flags & GV_INSUBGRAPH) == 0);
5710 
5711                         /*
5712                          * Services which are up need to come down before
5713                          * we're done, but we can only disable the leaves
5714                          * here.
5715                          */
5716 
5717                         if (up_state(v->gv_state))
5718                                 ++non_subgraph_svcs;
5719 
5720                         /* If it's already disabled, don't bother. */
5721                         if ((v->gv_flags & GV_ENABLED) == 0)
5722                                 continue;
5723 
5724                         if (!is_nonsubgraph_leaf(v))
5725                                 continue;
5726 
5727                         r = libscf_set_enable_ovr(inst, 0);
5728                         fs = "libscf_set_enable_ovr";
5729                 }
5730                 switch (r) {
5731                 case 0:
5732                 case ECANCELED:
5733                         break;
5734 
5735                 case ECONNABORTED:
5736                         libscf_handle_rebind(h);
5737                         rebound = B_TRUE;
5738                         goto again;
5739 
5740                 case EPERM:
5741                 case EROFS:
5742                         log_error(LOG_WARNING,
5743                             "Could not set %s/%s for %s: %s.\n",
5744                             SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
5745                             v->gv_name, strerror(r));
5746                         break;
5747 
5748                 default:
5749                         bad_error(fs, r);
5750                 }
5751         }
5752 
5753         if (halting != -1) {
5754                 if (non_subgraph_svcs > 1)
5755                         uu_warn("%d system services are now being stopped.\n",
5756                             non_subgraph_svcs);
5757                 else if (non_subgraph_svcs == 1)
5758                         uu_warn("One system service is now being stopped.\n");
5759                 else if (non_subgraph_svcs == 0)
5760                         do_uadmin();
5761         }
5762 
5763         ret = rebound ? ECONNRESET : 0;
5764 
5765 out:
5766         MUTEX_UNLOCK(&dgraph_lock);
5767         if (!isall && !isnone)
5768                 startd_free((void *)cfmri, max_scf_fmri_size);
5769         scf_instance_destroy(inst);
5770         return (ret);
5771 }
5772 
5773 
5774 /*
5775  * Returns 0, ECONNABORTED, or EINVAL.
5776  */
5777 static int
5778 handle_graph_update_event(scf_handle_t *h, graph_protocol_event_t *e)
5779 {
5780         int r;
5781 
5782         switch (e->gpe_type) {
5783         case GRAPH_UPDATE_RELOAD_GRAPH:
5784                 log_error(LOG_WARNING,
5785                     "graph_event: reload graph unimplemented\n");
5786                 break;
5787 
5788         case GRAPH_UPDATE_STATE_CHANGE: {
5789                 protocol_states_t *states = e->gpe_data;
5790 
5791                 switch (r = dgraph_set_instance_state(h, e->gpe_inst, states)) {
5792                 case 0:
5793                 case ENOENT:
5794                         break;
5795 
5796                 case ECONNABORTED:
5797                         return (ECONNABORTED);
5798 
5799                 case EINVAL:
5800                 default:
5801 #ifndef NDEBUG
5802                         (void) fprintf(stderr, "dgraph_set_instance_state() "
5803                             "failed with unexpected error %d at %s:%d.\n", r,
5804                             __FILE__, __LINE__);
5805 #endif
5806                         abort();
5807                 }
5808 
5809                 startd_free(states, sizeof (protocol_states_t));
5810                 break;
5811         }
5812 
5813         default:
5814                 log_error(LOG_WARNING,
5815                     "graph_event_loop received an unknown event: %d\n",
5816                     e->gpe_type);
5817                 break;
5818         }
5819 
5820         return (0);
5821 }
5822 
5823 /*
5824  * graph_event_thread()
5825  *    Wait for state changes from the restarters.
5826  */
5827 /*ARGSUSED*/
5828 void *
5829 graph_event_thread(void *unused)
5830 {
5831         scf_handle_t *h;
5832         int err;
5833 
5834         h = libscf_handle_create_bound_loop();
5835 
5836         /*CONSTCOND*/
5837         while (1) {
5838                 graph_protocol_event_t *e;
5839 
5840                 MUTEX_LOCK(&gu->gu_lock);
5841 
5842                 while (gu->gu_wakeup == 0)
5843                         (void) pthread_cond_wait(&gu->gu_cv, &gu->gu_lock);
5844 
5845                 gu->gu_wakeup = 0;
5846 
5847                 while ((e = graph_event_dequeue()) != NULL) {
5848                         MUTEX_LOCK(&e->gpe_lock);
5849                         MUTEX_UNLOCK(&gu->gu_lock);
5850 
5851                         while ((err = handle_graph_update_event(h, e)) ==
5852                             ECONNABORTED)
5853                                 libscf_handle_rebind(h);
5854 
5855                         if (err == 0)
5856                                 graph_event_release(e);
5857                         else
5858                                 graph_event_requeue(e);
5859 
5860                         MUTEX_LOCK(&gu->gu_lock);
5861                 }
5862 
5863                 MUTEX_UNLOCK(&gu->gu_lock);
5864         }
5865 
5866         /*
5867          * Unreachable for now -- there's currently no graceful cleanup
5868          * called on exit().
5869          */
5870         MUTEX_UNLOCK(&gu->gu_lock);
5871         scf_handle_destroy(h);
5872         return (NULL);
5873 }
5874 
5875 static void
5876 set_initial_milestone(scf_handle_t *h)
5877 {
5878         scf_instance_t *inst;
5879         char *fmri, *cfmri;
5880         size_t sz;
5881         int r;
5882 
5883         inst = safe_scf_instance_create(h);
5884         fmri = startd_alloc(max_scf_fmri_size);
5885 
5886         /*
5887          * If -m milestone= was specified, we want to set options_ovr/milestone
5888          * to it.  Otherwise we want to read what the milestone should be set
5889          * to.  Either way we need our inst.
5890          */
5891 get_self:
5892         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
5893             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5894                 switch (scf_error()) {
5895                 case SCF_ERROR_CONNECTION_BROKEN:
5896                         libscf_handle_rebind(h);
5897                         goto get_self;
5898 
5899                 case SCF_ERROR_NOT_FOUND:
5900                         if (st->st_subgraph != NULL &&
5901                             st->st_subgraph[0] != '\0') {
5902                                 sz = strlcpy(fmri, st->st_subgraph,
5903                                     max_scf_fmri_size);
5904                                 assert(sz < max_scf_fmri_size);
5905                         } else {
5906                                 fmri[0] = '\0';
5907                         }
5908                         break;
5909 
5910                 case SCF_ERROR_INVALID_ARGUMENT:
5911                 case SCF_ERROR_CONSTRAINT_VIOLATED:
5912                 case SCF_ERROR_HANDLE_MISMATCH:
5913                 default:
5914                         bad_error("scf_handle_decode_fmri", scf_error());
5915                 }
5916         } else {
5917                 if (st->st_subgraph != NULL && st->st_subgraph[0] != '\0') {
5918                         scf_propertygroup_t *pg;
5919 
5920                         pg = safe_scf_pg_create(h);
5921 
5922                         sz = strlcpy(fmri, st->st_subgraph, max_scf_fmri_size);
5923                         assert(sz < max_scf_fmri_size);
5924 
5925                         r = libscf_inst_get_or_add_pg(inst, SCF_PG_OPTIONS_OVR,
5926                             SCF_PG_OPTIONS_OVR_TYPE, SCF_PG_OPTIONS_OVR_FLAGS,
5927                             pg);
5928                         switch (r) {
5929                         case 0:
5930                                 break;
5931 
5932                         case ECONNABORTED:
5933                                 libscf_handle_rebind(h);
5934                                 goto get_self;
5935 
5936                         case EPERM:
5937                         case EACCES:
5938                         case EROFS:
5939                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5940                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5941                                     SCF_PROPERTY_MILESTONE, strerror(r));
5942                                 /* FALLTHROUGH */
5943 
5944                         case ECANCELED:
5945                                 sz = strlcpy(fmri, st->st_subgraph,
5946                                     max_scf_fmri_size);
5947                                 assert(sz < max_scf_fmri_size);
5948                                 break;
5949 
5950                         default:
5951                                 bad_error("libscf_inst_get_or_add_pg", r);
5952                         }
5953 
5954                         r = libscf_clear_runlevel(pg, fmri);
5955                         switch (r) {
5956                         case 0:
5957                                 break;
5958 
5959                         case ECONNABORTED:
5960                                 libscf_handle_rebind(h);
5961                                 goto get_self;
5962 
5963                         case EPERM:
5964                         case EACCES:
5965                         case EROFS:
5966                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5967                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5968                                     SCF_PROPERTY_MILESTONE, strerror(r));
5969                                 /* FALLTHROUGH */
5970 
5971                         case ECANCELED:
5972                                 sz = strlcpy(fmri, st->st_subgraph,
5973                                     max_scf_fmri_size);
5974                                 assert(sz < max_scf_fmri_size);
5975                                 break;
5976 
5977                         default:
5978                                 bad_error("libscf_clear_runlevel", r);
5979                         }
5980 
5981                         scf_pg_destroy(pg);
5982                 } else {
5983                         scf_property_t *prop;
5984                         scf_value_t *val;
5985 
5986                         prop = safe_scf_property_create(h);
5987                         val = safe_scf_value_create(h);
5988 
5989                         r = libscf_get_milestone(inst, prop, val, fmri,
5990                             max_scf_fmri_size);
5991                         switch (r) {
5992                         case 0:
5993                                 break;
5994 
5995                         case ECONNABORTED:
5996                                 libscf_handle_rebind(h);
5997                                 goto get_self;
5998 
5999                         case EINVAL:
6000                                 log_error(LOG_WARNING, "Milestone property is "
6001                                     "misconfigured.  Defaulting to \"all\".\n");
6002                                 /* FALLTHROUGH */
6003 
6004                         case ECANCELED:
6005                         case ENOENT:
6006                                 fmri[0] = '\0';
6007                                 break;
6008 
6009                         default:
6010                                 bad_error("libscf_get_milestone", r);
6011                         }
6012 
6013                         scf_value_destroy(val);
6014                         scf_property_destroy(prop);
6015                 }
6016         }
6017 
6018         if (fmri[0] == '\0' || strcmp(fmri, "all") == 0)
6019                 goto out;
6020 
6021         if (strcmp(fmri, "none") != 0) {
6022 retry:
6023                 if (scf_handle_decode_fmri(h, fmri, NULL, NULL, inst, NULL,
6024                     NULL, SCF_DECODE_FMRI_EXACT) != 0) {
6025                         switch (scf_error()) {
6026                         case SCF_ERROR_INVALID_ARGUMENT:
6027                                 log_error(LOG_WARNING,
6028                                     "Requested milestone \"%s\" is invalid.  "
6029                                     "Reverting to \"all\".\n", fmri);
6030                                 goto out;
6031 
6032                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6033                                 log_error(LOG_WARNING, "Requested milestone "
6034                                     "\"%s\" does not specify an instance.  "
6035                                     "Reverting to \"all\".\n", fmri);
6036                                 goto out;
6037 
6038                         case SCF_ERROR_CONNECTION_BROKEN:
6039                                 libscf_handle_rebind(h);
6040                                 goto retry;
6041 
6042                         case SCF_ERROR_NOT_FOUND:
6043                                 log_error(LOG_WARNING, "Requested milestone "
6044                                     "\"%s\" not in repository.  Reverting to "
6045                                     "\"all\".\n", fmri);
6046                                 goto out;
6047 
6048                         case SCF_ERROR_HANDLE_MISMATCH:
6049                         default:
6050                                 bad_error("scf_handle_decode_fmri",
6051                                     scf_error());
6052                         }
6053                 }
6054 
6055                 r = fmri_canonify(fmri, &cfmri, B_FALSE);
6056                 assert(r == 0);
6057 
6058                 r = dgraph_add_instance(cfmri, inst, B_TRUE);
6059                 startd_free(cfmri, max_scf_fmri_size);
6060                 switch (r) {
6061                 case 0:
6062                         break;
6063 
6064                 case ECONNABORTED:
6065                         goto retry;
6066 
6067                 case EINVAL:
6068                         log_error(LOG_WARNING,
6069                             "Requested milestone \"%s\" is invalid.  "
6070                             "Reverting to \"all\".\n", fmri);
6071                         goto out;
6072 
6073                 case ECANCELED:
6074                         log_error(LOG_WARNING,
6075                             "Requested milestone \"%s\" not "
6076                             "in repository.  Reverting to \"all\".\n",
6077                             fmri);
6078                         goto out;
6079 
6080                 case EEXIST:
6081                 default:
6082                         bad_error("dgraph_add_instance", r);
6083                 }
6084         }
6085 
6086         log_console(LOG_INFO, "Booting to milestone \"%s\".\n", fmri);
6087 
6088         r = dgraph_set_milestone(fmri, h, B_FALSE);
6089         switch (r) {
6090         case 0:
6091         case ECONNRESET:
6092         case EALREADY:
6093                 break;
6094 
6095         case EINVAL:
6096         case ENOENT:
6097         default:
6098                 bad_error("dgraph_set_milestone", r);
6099         }
6100 
6101 out:
6102         startd_free(fmri, max_scf_fmri_size);
6103         scf_instance_destroy(inst);
6104 }
6105 
6106 void
6107 set_restart_milestone(scf_handle_t *h)
6108 {
6109         scf_instance_t *inst;
6110         scf_property_t *prop;
6111         scf_value_t *val;
6112         char *fmri;
6113         int r;
6114 
6115         inst = safe_scf_instance_create(h);
6116 
6117 get_self:
6118         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL,
6119             inst, NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
6120                 switch (scf_error()) {
6121                 case SCF_ERROR_CONNECTION_BROKEN:
6122                         libscf_handle_rebind(h);
6123                         goto get_self;
6124 
6125                 case SCF_ERROR_NOT_FOUND:
6126                         break;
6127 
6128                 case SCF_ERROR_INVALID_ARGUMENT:
6129                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6130                 case SCF_ERROR_HANDLE_MISMATCH:
6131                 default:
6132                         bad_error("scf_handle_decode_fmri", scf_error());
6133                 }
6134 
6135                 scf_instance_destroy(inst);
6136                 return;
6137         }
6138 
6139         prop = safe_scf_property_create(h);
6140         val = safe_scf_value_create(h);
6141         fmri = startd_alloc(max_scf_fmri_size);
6142 
6143         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6144         switch (r) {
6145         case 0:
6146                 break;
6147 
6148         case ECONNABORTED:
6149                 libscf_handle_rebind(h);
6150                 goto get_self;
6151 
6152         case ECANCELED:
6153         case ENOENT:
6154         case EINVAL:
6155                 goto out;
6156 
6157         default:
6158                 bad_error("libscf_get_milestone", r);
6159         }
6160 
6161         r = dgraph_set_milestone(fmri, h, B_TRUE);
6162         switch (r) {
6163         case 0:
6164         case ECONNRESET:
6165         case EALREADY:
6166         case EINVAL:
6167         case ENOENT:
6168                 break;
6169 
6170         default:
6171                 bad_error("dgraph_set_milestone", r);
6172         }
6173 
6174 out:
6175         startd_free(fmri, max_scf_fmri_size);
6176         scf_value_destroy(val);
6177         scf_property_destroy(prop);
6178         scf_instance_destroy(inst);
6179 }
6180 
6181 /*
6182  * void *graph_thread(void *)
6183  *
6184  * Graph management thread.
6185  */
6186 /*ARGSUSED*/
6187 void *
6188 graph_thread(void *arg)
6189 {
6190         scf_handle_t *h;
6191         int err;
6192 
6193         h = libscf_handle_create_bound_loop();
6194 
6195         if (st->st_initial)
6196                 set_initial_milestone(h);
6197 
6198         MUTEX_LOCK(&dgraph_lock);
6199         initial_milestone_set = B_TRUE;
6200         err = pthread_cond_broadcast(&initial_milestone_cv);
6201         assert(err == 0);
6202         MUTEX_UNLOCK(&dgraph_lock);
6203 
6204         libscf_populate_graph(h);
6205 
6206         if (!st->st_initial)
6207                 set_restart_milestone(h);
6208 
6209         MUTEX_LOCK(&st->st_load_lock);
6210         st->st_load_complete = 1;
6211         (void) pthread_cond_broadcast(&st->st_load_cv);
6212         MUTEX_UNLOCK(&st->st_load_lock);
6213 
6214         MUTEX_LOCK(&dgraph_lock);
6215         /*
6216          * Now that we've set st_load_complete we need to check can_come_up()
6217          * since if we booted to a milestone, then there won't be any more
6218          * state updates.
6219          */
6220         if (!go_single_user_mode && !go_to_level1 &&
6221             halting == -1) {
6222                 if (!sulogin_thread_running && !can_come_up()) {
6223                         (void) startd_thread_create(sulogin_thread, NULL);
6224                         sulogin_thread_running = B_TRUE;
6225                 }
6226         }
6227         MUTEX_UNLOCK(&dgraph_lock);
6228 
6229         (void) pthread_mutex_lock(&gu->gu_freeze_lock);
6230 
6231         /*CONSTCOND*/
6232         while (1) {
6233                 (void) pthread_cond_wait(&gu->gu_freeze_cv,
6234                     &gu->gu_freeze_lock);
6235         }
6236 
6237         /*
6238          * Unreachable for now -- there's currently no graceful cleanup
6239          * called on exit().
6240          */
6241         (void) pthread_mutex_unlock(&gu->gu_freeze_lock);
6242         scf_handle_destroy(h);
6243 
6244         return (NULL);
6245 }
6246 
6247 
6248 /*
6249  * int next_action()
6250  *   Given an array of timestamps 'a' with 'num' elements, find the
6251  *   lowest non-zero timestamp and return its index. If there are no
6252  *   non-zero elements, return -1.
6253  */
6254 static int
6255 next_action(hrtime_t *a, int num)
6256 {
6257         hrtime_t t = 0;
6258         int i = 0, smallest = -1;
6259 
6260         for (i = 0; i < num; i++) {
6261                 if (t == 0) {
6262                         t = a[i];
6263                         smallest = i;
6264                 } else if (a[i] != 0 && a[i] < t) {
6265                         t = a[i];
6266                         smallest = i;
6267                 }
6268         }
6269 
6270         if (t == 0)
6271                 return (-1);
6272         else
6273                 return (smallest);
6274 }
6275 
6276 /*
6277  * void process_actions()
6278  *   Process actions requested by the administrator. Possibilities include:
6279  *   refresh, restart, maintenance mode off, maintenance mode on,
6280  *   maintenance mode immediate, and degraded.
6281  *
6282  *   The set of pending actions is represented in the repository as a
6283  *   per-instance property group, with each action being a single property
6284  *   in that group.  This property group is converted to an array, with each
6285  *   action type having an array slot.  The actions in the array at the
6286  *   time process_actions() is called are acted on in the order of the
6287  *   timestamp (which is the value stored in the slot).  A value of zero
6288  *   indicates that there is no pending action of the type associated with
6289  *   a particular slot.
6290  *
6291  *   Sending an action event multiple times before the restarter has a
6292  *   chance to process that action will force it to be run at the last
6293  *   timestamp where it appears in the ordering.
6294  *
6295  *   Turning maintenance mode on trumps all other actions.
6296  *
6297  *   Returns 0 or ECONNABORTED.
6298  */
6299 static int
6300 process_actions(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst)
6301 {
6302         scf_property_t *prop = NULL;
6303         scf_value_t *val = NULL;
6304         scf_type_t type;
6305         graph_vertex_t *vertex;
6306         admin_action_t a;
6307         int i, ret = 0, r;
6308         hrtime_t action_ts[NACTIONS];
6309         char *inst_name;
6310 
6311         r = libscf_instance_get_fmri(inst, &inst_name);
6312         switch (r) {
6313         case 0:
6314                 break;
6315 
6316         case ECONNABORTED:
6317                 return (ECONNABORTED);
6318 
6319         case ECANCELED:
6320                 return (0);
6321 
6322         default:
6323                 bad_error("libscf_instance_get_fmri", r);
6324         }
6325 
6326         MUTEX_LOCK(&dgraph_lock);
6327 
6328         vertex = vertex_get_by_name(inst_name);
6329         if (vertex == NULL) {
6330                 MUTEX_UNLOCK(&dgraph_lock);
6331                 log_framework(LOG_DEBUG, "%s: Can't find graph vertex. "
6332                     "The instance must have been removed.\n", inst_name);
6333                 startd_free(inst_name, max_scf_fmri_size);
6334                 return (0);
6335         }
6336 
6337         prop = safe_scf_property_create(h);
6338         val = safe_scf_value_create(h);
6339 
6340         for (i = 0; i < NACTIONS; i++) {
6341                 if (scf_pg_get_property(pg, admin_actions[i], prop) != 0) {
6342                         switch (scf_error()) {
6343                         case SCF_ERROR_CONNECTION_BROKEN:
6344                         default:
6345                                 ret = ECONNABORTED;
6346                                 goto out;
6347 
6348                         case SCF_ERROR_DELETED:
6349                                 goto out;
6350 
6351                         case SCF_ERROR_NOT_FOUND:
6352                                 action_ts[i] = 0;
6353                                 continue;
6354 
6355                         case SCF_ERROR_HANDLE_MISMATCH:
6356                         case SCF_ERROR_INVALID_ARGUMENT:
6357                         case SCF_ERROR_NOT_SET:
6358                                 bad_error("scf_pg_get_property", scf_error());
6359                         }
6360                 }
6361 
6362                 if (scf_property_type(prop, &type) != 0) {
6363                         switch (scf_error()) {
6364                         case SCF_ERROR_CONNECTION_BROKEN:
6365                         default:
6366                                 ret = ECONNABORTED;
6367                                 goto out;
6368 
6369                         case SCF_ERROR_DELETED:
6370                                 action_ts[i] = 0;
6371                                 continue;
6372 
6373                         case SCF_ERROR_NOT_SET:
6374                                 bad_error("scf_property_type", scf_error());
6375                         }
6376                 }
6377 
6378                 if (type != SCF_TYPE_INTEGER) {
6379                         action_ts[i] = 0;
6380                         continue;
6381                 }
6382 
6383                 if (scf_property_get_value(prop, val) != 0) {
6384                         switch (scf_error()) {
6385                         case SCF_ERROR_CONNECTION_BROKEN:
6386                         default:
6387                                 ret = ECONNABORTED;
6388                                 goto out;
6389 
6390                         case SCF_ERROR_DELETED:
6391                                 goto out;
6392 
6393                         case SCF_ERROR_NOT_FOUND:
6394                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6395                                 action_ts[i] = 0;
6396                                 continue;
6397 
6398                         case SCF_ERROR_NOT_SET:
6399                         case SCF_ERROR_PERMISSION_DENIED:
6400                                 bad_error("scf_property_get_value",
6401                                     scf_error());
6402                         }
6403                 }
6404 
6405                 r = scf_value_get_integer(val, &action_ts[i]);
6406                 assert(r == 0);
6407         }
6408 
6409         a = ADMIN_EVENT_MAINT_ON_IMMEDIATE;
6410         if (action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ||
6411             action_ts[ADMIN_EVENT_MAINT_ON]) {
6412                 a = action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ?
6413                     ADMIN_EVENT_MAINT_ON_IMMEDIATE : ADMIN_EVENT_MAINT_ON;
6414 
6415                 vertex_send_event(vertex, admin_events[a]);
6416                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6417                 switch (r) {
6418                 case 0:
6419                 case EACCES:
6420                         break;
6421 
6422                 case ECONNABORTED:
6423                         ret = ECONNABORTED;
6424                         goto out;
6425 
6426                 case EPERM:
6427                         uu_die("Insufficient privilege.\n");
6428                         /* NOTREACHED */
6429 
6430                 default:
6431                         bad_error("libscf_unset_action", r);
6432                 }
6433         }
6434 
6435         while ((a = next_action(action_ts, NACTIONS)) != -1) {
6436                 log_framework(LOG_DEBUG,
6437                     "Graph: processing %s action for %s.\n", admin_actions[a],
6438                     inst_name);
6439 
6440                 if (a == ADMIN_EVENT_REFRESH) {
6441                         r = dgraph_refresh_instance(vertex, inst);
6442                         switch (r) {
6443                         case 0:
6444                         case ECANCELED:
6445                         case EINVAL:
6446                         case -1:
6447                                 break;
6448 
6449                         case ECONNABORTED:
6450                                 /* pg & inst are reset now, so just return. */
6451                                 ret = ECONNABORTED;
6452                                 goto out;
6453 
6454                         default:
6455                                 bad_error("dgraph_refresh_instance", r);
6456                         }
6457                 }
6458 
6459                 vertex_send_event(vertex, admin_events[a]);
6460 
6461                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6462                 switch (r) {
6463                 case 0:
6464                 case EACCES:
6465                         break;
6466 
6467                 case ECONNABORTED:
6468                         ret = ECONNABORTED;
6469                         goto out;
6470 
6471                 case EPERM:
6472                         uu_die("Insufficient privilege.\n");
6473                         /* NOTREACHED */
6474 
6475                 default:
6476                         bad_error("libscf_unset_action", r);
6477                 }
6478 
6479                 action_ts[a] = 0;
6480         }
6481 
6482 out:
6483         MUTEX_UNLOCK(&dgraph_lock);
6484 
6485         scf_property_destroy(prop);
6486         scf_value_destroy(val);
6487         startd_free(inst_name, max_scf_fmri_size);
6488         return (ret);
6489 }
6490 
6491 /*
6492  * inst and pg_name are scratch space, and are unset on entry.
6493  * Returns
6494  *   0 - success
6495  *   ECONNRESET - success, but repository handle rebound
6496  *   ECONNABORTED - repository connection broken
6497  */
6498 static int
6499 process_pg_event(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst,
6500     char *pg_name)
6501 {
6502         int r;
6503         scf_property_t *prop;
6504         scf_value_t *val;
6505         char *fmri;
6506         boolean_t rebound = B_FALSE, rebind_inst = B_FALSE;
6507 
6508         if (scf_pg_get_name(pg, pg_name, max_scf_value_size) < 0) {
6509                 switch (scf_error()) {
6510                 case SCF_ERROR_CONNECTION_BROKEN:
6511                 default:
6512                         return (ECONNABORTED);
6513 
6514                 case SCF_ERROR_DELETED:
6515                         return (0);
6516 
6517                 case SCF_ERROR_NOT_SET:
6518                         bad_error("scf_pg_get_name", scf_error());
6519                 }
6520         }
6521 
6522         if (strcmp(pg_name, SCF_PG_GENERAL) == 0 ||
6523             strcmp(pg_name, SCF_PG_GENERAL_OVR) == 0) {
6524                 r = dgraph_update_general(pg);
6525                 switch (r) {
6526                 case 0:
6527                 case ENOTSUP:
6528                 case ECANCELED:
6529                         return (0);
6530 
6531                 case ECONNABORTED:
6532                         return (ECONNABORTED);
6533 
6534                 case -1:
6535                         /* Error should have been logged. */
6536                         return (0);
6537 
6538                 default:
6539                         bad_error("dgraph_update_general", r);
6540                 }
6541         } else if (strcmp(pg_name, SCF_PG_RESTARTER_ACTIONS) == 0) {
6542                 if (scf_pg_get_parent_instance(pg, inst) != 0) {
6543                         switch (scf_error()) {
6544                         case SCF_ERROR_CONNECTION_BROKEN:
6545                                 return (ECONNABORTED);
6546 
6547                         case SCF_ERROR_DELETED:
6548                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6549                                 /* Ignore commands on services. */
6550                                 return (0);
6551 
6552                         case SCF_ERROR_NOT_BOUND:
6553                         case SCF_ERROR_HANDLE_MISMATCH:
6554                         case SCF_ERROR_NOT_SET:
6555                         default:
6556                                 bad_error("scf_pg_get_parent_instance",
6557                                     scf_error());
6558                         }
6559                 }
6560 
6561                 return (process_actions(h, pg, inst));
6562         }
6563 
6564         if (strcmp(pg_name, SCF_PG_OPTIONS) != 0 &&
6565             strcmp(pg_name, SCF_PG_OPTIONS_OVR) != 0)
6566                 return (0);
6567 
6568         /*
6569          * We only care about the options[_ovr] property groups of our own
6570          * instance, so get the fmri and compare.  Plus, once we know it's
6571          * correct, if the repository connection is broken we know exactly what
6572          * property group we were operating on, and can look it up again.
6573          */
6574         if (scf_pg_get_parent_instance(pg, inst) != 0) {
6575                 switch (scf_error()) {
6576                 case SCF_ERROR_CONNECTION_BROKEN:
6577                         return (ECONNABORTED);
6578 
6579                 case SCF_ERROR_DELETED:
6580                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6581                         return (0);
6582 
6583                 case SCF_ERROR_HANDLE_MISMATCH:
6584                 case SCF_ERROR_NOT_BOUND:
6585                 case SCF_ERROR_NOT_SET:
6586                 default:
6587                         bad_error("scf_pg_get_parent_instance",
6588                             scf_error());
6589                 }
6590         }
6591 
6592         switch (r = libscf_instance_get_fmri(inst, &fmri)) {
6593         case 0:
6594                 break;
6595 
6596         case ECONNABORTED:
6597                 return (ECONNABORTED);
6598 
6599         case ECANCELED:
6600                 return (0);
6601 
6602         default:
6603                 bad_error("libscf_instance_get_fmri", r);
6604         }
6605 
6606         if (strcmp(fmri, SCF_SERVICE_STARTD) != 0) {
6607                 startd_free(fmri, max_scf_fmri_size);
6608                 return (0);
6609         }
6610 
6611         /*
6612          * update the information events flag
6613          */
6614         if (strcmp(pg_name, SCF_PG_OPTIONS) == 0)
6615                 info_events_all = libscf_get_info_events_all(pg);
6616 
6617         prop = safe_scf_property_create(h);
6618         val = safe_scf_value_create(h);
6619 
6620         if (strcmp(pg_name, SCF_PG_OPTIONS_OVR) == 0) {
6621                 /* See if we need to set the runlevel. */
6622                 /* CONSTCOND */
6623                 if (0) {
6624 rebind_pg:
6625                         libscf_handle_rebind(h);
6626                         rebound = B_TRUE;
6627 
6628                         r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6629                         switch (r) {
6630                         case 0:
6631                                 break;
6632 
6633                         case ECONNABORTED:
6634                                 goto rebind_pg;
6635 
6636                         case ENOENT:
6637                                 goto out;
6638 
6639                         case EINVAL:
6640                         case ENOTSUP:
6641                                 bad_error("libscf_lookup_instance", r);
6642                         }
6643 
6644                         if (scf_instance_get_pg(inst, pg_name, pg) != 0) {
6645                                 switch (scf_error()) {
6646                                 case SCF_ERROR_DELETED:
6647                                 case SCF_ERROR_NOT_FOUND:
6648                                         goto out;
6649 
6650                                 case SCF_ERROR_CONNECTION_BROKEN:
6651                                         goto rebind_pg;
6652 
6653                                 case SCF_ERROR_HANDLE_MISMATCH:
6654                                 case SCF_ERROR_NOT_BOUND:
6655                                 case SCF_ERROR_NOT_SET:
6656                                 case SCF_ERROR_INVALID_ARGUMENT:
6657                                 default:
6658                                         bad_error("scf_instance_get_pg",
6659                                             scf_error());
6660                                 }
6661                         }
6662                 }
6663 
6664                 if (scf_pg_get_property(pg, "runlevel", prop) == 0) {
6665                         r = dgraph_set_runlevel(pg, prop);
6666                         switch (r) {
6667                         case ECONNRESET:
6668                                 rebound = B_TRUE;
6669                                 rebind_inst = B_TRUE;
6670                                 /* FALLTHROUGH */
6671 
6672                         case 0:
6673                                 break;
6674 
6675                         case ECONNABORTED:
6676                                 goto rebind_pg;
6677 
6678                         case ECANCELED:
6679                                 goto out;
6680 
6681                         default:
6682                                 bad_error("dgraph_set_runlevel", r);
6683                         }
6684                 } else {
6685                         switch (scf_error()) {
6686                         case SCF_ERROR_CONNECTION_BROKEN:
6687                         default:
6688                                 goto rebind_pg;
6689 
6690                         case SCF_ERROR_DELETED:
6691                                 goto out;
6692 
6693                         case SCF_ERROR_NOT_FOUND:
6694                                 break;
6695 
6696                         case SCF_ERROR_INVALID_ARGUMENT:
6697                         case SCF_ERROR_HANDLE_MISMATCH:
6698                         case SCF_ERROR_NOT_BOUND:
6699                         case SCF_ERROR_NOT_SET:
6700                                 bad_error("scf_pg_get_property", scf_error());
6701                         }
6702                 }
6703         }
6704 
6705         if (rebind_inst) {
6706 lookup_inst:
6707                 r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6708                 switch (r) {
6709                 case 0:
6710                         break;
6711 
6712                 case ECONNABORTED:
6713                         libscf_handle_rebind(h);
6714                         rebound = B_TRUE;
6715                         goto lookup_inst;
6716 
6717                 case ENOENT:
6718                         goto out;
6719 
6720                 case EINVAL:
6721                 case ENOTSUP:
6722                         bad_error("libscf_lookup_instance", r);
6723                 }
6724         }
6725 
6726         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6727         switch (r) {
6728         case 0:
6729                 break;
6730 
6731         case ECONNABORTED:
6732                 libscf_handle_rebind(h);
6733                 rebound = B_TRUE;
6734                 goto lookup_inst;
6735 
6736         case EINVAL:
6737                 log_error(LOG_NOTICE,
6738                     "%s/%s property of %s is misconfigured.\n", pg_name,
6739                     SCF_PROPERTY_MILESTONE, SCF_SERVICE_STARTD);
6740                 /* FALLTHROUGH */
6741 
6742         case ECANCELED:
6743         case ENOENT:
6744                 (void) strcpy(fmri, "all");
6745                 break;
6746 
6747         default:
6748                 bad_error("libscf_get_milestone", r);
6749         }
6750 
6751         r = dgraph_set_milestone(fmri, h, B_FALSE);
6752         switch (r) {
6753         case 0:
6754         case ECONNRESET:
6755         case EALREADY:
6756                 break;
6757 
6758         case EINVAL:
6759                 log_error(LOG_WARNING, "Milestone %s is invalid.\n", fmri);
6760                 break;
6761 
6762         case ENOENT:
6763                 log_error(LOG_WARNING, "Milestone %s does not exist.\n", fmri);
6764                 break;
6765 
6766         default:
6767                 bad_error("dgraph_set_milestone", r);
6768         }
6769 
6770 out:
6771         startd_free(fmri, max_scf_fmri_size);
6772         scf_value_destroy(val);
6773         scf_property_destroy(prop);
6774 
6775         return (rebound ? ECONNRESET : 0);
6776 }
6777 
6778 /*
6779  * process_delete() deletes an instance from the dgraph if 'fmri' is an
6780  * instance fmri or if 'fmri' matches the 'general' property group of an
6781  * instance (or the 'general/enabled' property).
6782  *
6783  * 'fmri' may be overwritten and cannot be trusted on return by the caller.
6784  */
6785 static void
6786 process_delete(char *fmri, scf_handle_t *h)
6787 {
6788         char *lfmri, *end_inst_fmri;
6789         const char *inst_name = NULL;
6790         const char *pg_name = NULL;
6791         const char *prop_name = NULL;
6792 
6793         lfmri = safe_strdup(fmri);
6794 
6795         /* Determine if the FMRI is a property group or instance */
6796         if (scf_parse_svc_fmri(lfmri, NULL, NULL, &inst_name, &pg_name,
6797             &prop_name) != SCF_SUCCESS) {
6798                 log_error(LOG_WARNING,
6799                     "Received invalid FMRI \"%s\" from repository server.\n",
6800                     fmri);
6801         } else if (inst_name != NULL && pg_name == NULL) {
6802                 (void) dgraph_remove_instance(fmri, h);
6803         } else if (inst_name != NULL && pg_name != NULL) {
6804                 /*
6805                  * If we're deleting the 'general' property group or
6806                  * 'general/enabled' property then the whole instance
6807                  * must be removed from the dgraph.
6808                  */
6809                 if (strcmp(pg_name, SCF_PG_GENERAL) != 0) {
6810                         free(lfmri);
6811                         return;
6812                 }
6813 
6814                 if (prop_name != NULL &&
6815                     strcmp(prop_name, SCF_PROPERTY_ENABLED) != 0) {
6816                         free(lfmri);
6817                         return;
6818                 }
6819 
6820                 /*
6821                  * Because the instance has already been deleted from the
6822                  * repository, we cannot use any scf_ functions to retrieve
6823                  * the instance FMRI however we can easily reconstruct it
6824                  * manually.
6825                  */
6826                 end_inst_fmri = strstr(fmri, SCF_FMRI_PROPERTYGRP_PREFIX);
6827                 if (end_inst_fmri == NULL)
6828                         bad_error("process_delete", 0);
6829 
6830                 end_inst_fmri[0] = '\0';
6831 
6832                 (void) dgraph_remove_instance(fmri, h);
6833         }
6834 
6835         free(lfmri);
6836 }
6837 
6838 /*ARGSUSED*/
6839 void *
6840 repository_event_thread(void *unused)
6841 {
6842         scf_handle_t *h;
6843         scf_propertygroup_t *pg;
6844         scf_instance_t *inst;
6845         char *fmri = startd_alloc(max_scf_fmri_size);
6846         char *pg_name = startd_alloc(max_scf_value_size);
6847         int r;
6848 
6849         h = libscf_handle_create_bound_loop();
6850 
6851         pg = safe_scf_pg_create(h);
6852         inst = safe_scf_instance_create(h);
6853 
6854 retry:
6855         if (_scf_notify_add_pgtype(h, SCF_GROUP_FRAMEWORK) != SCF_SUCCESS) {
6856                 if (scf_error() == SCF_ERROR_CONNECTION_BROKEN) {
6857                         libscf_handle_rebind(h);
6858                 } else {
6859                         log_error(LOG_WARNING,
6860                             "Couldn't set up repository notification "
6861                             "for property group type %s: %s\n",
6862                             SCF_GROUP_FRAMEWORK, scf_strerror(scf_error()));
6863 
6864                         (void) sleep(1);
6865                 }
6866 
6867                 goto retry;
6868         }
6869 
6870         /*CONSTCOND*/
6871         while (1) {
6872                 ssize_t res;
6873 
6874                 /* Note: fmri is only set on delete events. */
6875                 res = _scf_notify_wait(pg, fmri, max_scf_fmri_size);
6876                 if (res < 0) {
6877                         libscf_handle_rebind(h);
6878                         goto retry;
6879                 } else if (res == 0) {
6880                         /*
6881                          * property group modified.  inst and pg_name are
6882                          * pre-allocated scratch space.
6883                          */
6884                         if (scf_pg_update(pg) < 0) {
6885                                 switch (scf_error()) {
6886                                 case SCF_ERROR_DELETED:
6887                                         continue;
6888 
6889                                 case SCF_ERROR_CONNECTION_BROKEN:
6890                                         log_error(LOG_WARNING,
6891                                             "Lost repository event due to "
6892                                             "disconnection.\n");
6893                                         libscf_handle_rebind(h);
6894                                         goto retry;
6895 
6896                                 case SCF_ERROR_NOT_BOUND:
6897                                 case SCF_ERROR_NOT_SET:
6898                                 default:
6899                                         bad_error("scf_pg_update", scf_error());
6900                                 }
6901                         }
6902 
6903                         r = process_pg_event(h, pg, inst, pg_name);
6904                         switch (r) {
6905                         case 0:
6906                                 break;
6907 
6908                         case ECONNABORTED:
6909                                 log_error(LOG_WARNING, "Lost repository event "
6910                                     "due to disconnection.\n");
6911                                 libscf_handle_rebind(h);
6912                                 /* FALLTHROUGH */
6913 
6914                         case ECONNRESET:
6915                                 goto retry;
6916 
6917                         default:
6918                                 bad_error("process_pg_event", r);
6919                         }
6920                 } else {
6921                         /*
6922                          * Service, instance, or pg deleted.
6923                          * Don't trust fmri on return.
6924                          */
6925                         process_delete(fmri, h);
6926                 }
6927         }
6928 
6929         /*NOTREACHED*/
6930         return (NULL);
6931 }
6932 
6933 void
6934 graph_engine_start()
6935 {
6936         int err;
6937 
6938         (void) startd_thread_create(graph_thread, NULL);
6939 
6940         MUTEX_LOCK(&dgraph_lock);
6941         while (!initial_milestone_set) {
6942                 err = pthread_cond_wait(&initial_milestone_cv, &dgraph_lock);
6943                 assert(err == 0);
6944         }
6945         MUTEX_UNLOCK(&dgraph_lock);
6946 
6947         (void) startd_thread_create(repository_event_thread, NULL);
6948         (void) startd_thread_create(graph_event_thread, NULL);
6949 }