1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
  25  */
  26 
  27 /*
  28  * graph.c - master restarter graph engine
  29  *
  30  *   The graph engine keeps a dependency graph of all service instances on the
  31  *   system, as recorded in the repository.  It decides when services should
  32  *   be brought up or down based on service states and dependencies and sends
  33  *   commands to restarters to effect any changes.  It also executes
  34  *   administrator commands sent by svcadm via the repository.
  35  *
  36  *   The graph is stored in uu_list_t *dgraph and its vertices are
  37  *   graph_vertex_t's, each of which has a name and an integer id unique to
  38  *   its name (see dict.c).  A vertex's type attribute designates the type
  39  *   of object it represents: GVT_INST for service instances, GVT_SVC for
  40  *   service objects (since service instances may depend on another service,
  41  *   rather than service instance), GVT_FILE for files (which services may
  42  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
  43  *   vertices are necessary because dependency lists may have particular
  44  *   grouping types (require any, require all, optional, or exclude) and
  45  *   event-propagation characteristics.
  46  *
  47  *   The initial graph is built by libscf_populate_graph() invoking
  48  *   dgraph_add_instance() for each instance in the repository.  The function
  49  *   adds a GVT_SVC vertex for the service if one does not already exist, adds
  50  *   a GVT_INST vertex named by the FMRI of the instance, and sets up the edges.
  51  *   The resulting web of vertices & edges associated with an instance's vertex
  52  *   includes
  53  *
  54  *     - an edge from the GVT_SVC vertex for the instance's service
  55  *
  56  *     - an edge to the GVT_INST vertex of the instance's resarter, if its
  57  *       restarter is not svc.startd
  58  *
  59  *     - edges from other GVT_INST vertices if the instance is a restarter
  60  *
  61  *     - for each dependency property group in the instance's "running"
  62  *       snapshot, an edge to a GVT_GROUP vertex named by the FMRI of the
  63  *       instance and the name of the property group
  64  *
  65  *     - for each value of the "entities" property in each dependency property
  66  *       group, an edge from the corresponding GVT_GROUP vertex to a
  67  *       GVT_INST, GVT_SVC, or GVT_FILE vertex
  68  *
  69  *     - edges from GVT_GROUP vertices for each dependent instance
  70  *
  71  *   After the edges are set up the vertex's GV_CONFIGURED flag is set.  If
  72  *   there are problems, or if a service is mentioned in a dependency but does
  73  *   not exist in the repository, the GV_CONFIGURED flag will be clear.
  74  *
  75  *   The graph and all of its vertices are protected by the dgraph_lock mutex.
  76  *   See restarter.c for more information.
  77  *
  78  *   The properties of an instance fall into two classes: immediate and
  79  *   snapshotted.  Immediate properties should have an immediate effect when
  80  *   changed.  Snapshotted properties should be read from a snapshot, so they
  81  *   only change when the snapshot changes.  The immediate properties used by
  82  *   the graph engine are general/enabled, general/restarter, and the properties
  83  *   in the restarter_actions property group.  Since they are immediate, they
  84  *   are not read out of a snapshot.  The snapshotted properties used by the
  85  *   graph engine are those in the property groups with type "dependency" and
  86  *   are read out of the "running" snapshot.  The "running" snapshot is created
  87  *   by the the graph engine as soon as possible, and it is updated, along with
  88  *   in-core copies of the data (dependency information for the graph engine) on
  89  *   receipt of the refresh command from svcadm.  In addition, the graph engine
  90  *   updates the "start" snapshot from the "running" snapshot whenever a service
  91  *   comes online.
  92  *
  93  *   When a DISABLE event is requested by the administrator, svc.startd shutdown
  94  *   the dependents first before shutting down the requested service.
  95  *   In graph_enable_by_vertex, we create a subtree that contains the dependent
  96  *   vertices by marking those vertices with the GV_TOOFFLINE flag. And we mark
  97  *   the vertex to disable with the GV_TODISABLE flag. Once the tree is created,
  98  *   we send the _ADMIN_DISABLE event to the leaves. The leaves will then
  99  *   transition from STATE_ONLINE/STATE_DEGRADED to STATE_OFFLINE/STATE_MAINT.
 100  *   In gt_enter_offline and gt_enter_maint if the vertex was in a subtree then
 101  *   we clear the GV_TOOFFLINE flag and walk the dependencies to offline the new
 102  *   exposed leaves. We do the same until we reach the last leaf (the one with
 103  *   the GV_TODISABLE flag). If the vertex to disable is also part of a larger
 104  *   subtree (eg. multiple DISABLE events on vertices in the same subtree) then
 105  *   once the first vertex is disabled (GV_TODISABLE flag is removed), we
 106  *   continue to propagate the offline event to the vertex's dependencies.
 107  *
 108  *
 109  * SMF state transition notifications
 110  *
 111  *   When an instance of a service managed by SMF changes state, svc.startd may
 112  *   publish a GPEC sysevent. All transitions to or from maintenance, a
 113  *   transition cause by a hardware error will generate an event.
 114  *   Other transitions will generate an event if there exist notification
 115  *   parameter for that transition. Notification parameters are stored in the
 116  *   SMF repository for the service/instance they refer to. System-wide
 117  *   notification parameters are stored in the global instance.
 118  *   svc.startd can be told to send events for all SMF state transitions despite
 119  *   of notification parameters by setting options/info_events_all to true in
 120  *   restarter:default
 121  *
 122  *   The set of transitions that generate events is cached in the
 123  *   dgraph_vertex_t gv_stn_tset for service/instance and in the global
 124  *   stn_global for the system-wide set. They are re-read when instances are
 125  *   refreshed.
 126  *
 127  *   The GPEC events published by svc.startd are consumed by fmd(1M). After
 128  *   processing these events, fmd(1M) publishes the processed events to
 129  *   notification agents. The notification agents read the notification
 130  *   parameters from the SMF repository through libscf(3LIB) interfaces and send
 131  *   the notification, or not, based on those parameters.
 132  *
 133  *   Subscription and publishing to the GPEC channels is done with the
 134  *   libfmevent(3LIB) wrappers fmev_[r]publish_*() and
 135  *   fmev_shdl_(un)subscribe().
 136  *
 137  */
 138 
 139 #include <sys/uadmin.h>
 140 #include <sys/wait.h>
 141 
 142 #include <assert.h>
 143 #include <errno.h>
 144 #include <fcntl.h>
 145 #include <fm/libfmevent.h>
 146 #include <libscf.h>
 147 #include <libscf_priv.h>
 148 #include <librestart.h>
 149 #include <libuutil.h>
 150 #include <locale.h>
 151 #include <poll.h>
 152 #include <pthread.h>
 153 #include <signal.h>
 154 #include <stddef.h>
 155 #include <stdio.h>
 156 #include <stdlib.h>
 157 #include <string.h>
 158 #include <strings.h>
 159 #include <sys/statvfs.h>
 160 #include <sys/uadmin.h>
 161 #include <zone.h>
 162 #if defined(__i386)
 163 #include <libgrubmgmt.h>
 164 #endif  /* __i386 */
 165 
 166 #include "startd.h"
 167 #include "protocol.h"
 168 
 169 
 170 #define MILESTONE_NONE  ((graph_vertex_t *)1)
 171 
 172 #define CONSOLE_LOGIN_FMRI      "svc:/system/console-login:default"
 173 #define FS_MINIMAL_FMRI         "svc:/system/filesystem/minimal:default"
 174 
 175 #define VERTEX_REMOVED  0       /* vertex has been freed  */
 176 #define VERTEX_INUSE    1       /* vertex is still in use */
 177 
 178 #define IS_ENABLED(v) ((v)->gv_flags & (GV_ENABLED | GV_ENBLD_NOOVR))
 179 
 180 /*
 181  * stn_global holds the tset for the system wide notification parameters.
 182  * It is updated on refresh of svc:/system/svc/global:default
 183  *
 184  * There are two assumptions that relax the need for a mutex:
 185  *     1. 32-bit value assignments are atomic
 186  *     2. Its value is consumed only in one point at
 187  *     dgraph_state_transition_notify(). There are no test and set races.
 188  *
 189  *     If either assumption is broken, we'll need a mutex to synchronize
 190  *     access to stn_global
 191  */
 192 int32_t stn_global;
 193 /*
 194  * info_events_all holds a flag to override notification parameters and send
 195  * Information events for all state transitions.
 196  * same about the need of a mutex here.
 197  */
 198 int info_events_all;
 199 
 200 /*
 201  * Services in these states are not considered 'down' by the
 202  * milestone/shutdown code.
 203  */
 204 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
 205         (state) == RESTARTER_STATE_DEGRADED || \
 206         (state) == RESTARTER_STATE_OFFLINE)
 207 
 208 #define is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
 209         ((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \
 210         (v->gv_depgroup == DEPGRP_OPTIONAL_ALL) || \
 211         (v->gv_restart < RERR_RESTART)))
 212 
 213 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
 214 static uu_list_t *dgraph;
 215 static pthread_mutex_t dgraph_lock;
 216 
 217 /*
 218  * milestone indicates the current subgraph.  When NULL, it is the entire
 219  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
 220  * services on which the target vertex depends.
 221  */
 222 static graph_vertex_t *milestone = NULL;
 223 static boolean_t initial_milestone_set = B_FALSE;
 224 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
 225 
 226 /* protected by dgraph_lock */
 227 static boolean_t sulogin_thread_running = B_FALSE;
 228 static boolean_t sulogin_running = B_FALSE;
 229 static boolean_t console_login_ready = B_FALSE;
 230 
 231 /* Number of services to come down to complete milestone transition. */
 232 static uint_t non_subgraph_svcs;
 233 
 234 /*
 235  * These variables indicate what should be done when we reach the milestone
 236  * target milestone, i.e., when non_subgraph_svcs == 0.  They are acted upon in
 237  * dgraph_set_instance_state().
 238  */
 239 static int halting = -1;
 240 static boolean_t go_single_user_mode = B_FALSE;
 241 static boolean_t go_to_level1 = B_FALSE;
 242 
 243 /*
 244  * Tracks when we started halting.
 245  */
 246 static time_t halting_time = 0;
 247 
 248 /*
 249  * This tracks the legacy runlevel to ensure we signal init and manage
 250  * utmpx entries correctly.
 251  */
 252 static char current_runlevel = '\0';
 253 
 254 /* Number of single user threads currently running */
 255 static pthread_mutex_t single_user_thread_lock;
 256 static int single_user_thread_count = 0;
 257 
 258 /* Statistics for dependency cycle-checking */
 259 static u_longlong_t dep_inserts = 0;
 260 static u_longlong_t dep_cycle_ns = 0;
 261 static u_longlong_t dep_insert_ns = 0;
 262 
 263 
 264 static const char * const emsg_invalid_restarter =
 265         "Transitioning %s to maintenance, restarter FMRI %s is invalid "
 266         "(see 'svcs -xv' for details).\n";
 267 static const char * const console_login_fmri = CONSOLE_LOGIN_FMRI;
 268 static const char * const single_user_fmri = SCF_MILESTONE_SINGLE_USER;
 269 static const char * const multi_user_fmri = SCF_MILESTONE_MULTI_USER;
 270 static const char * const multi_user_svr_fmri = SCF_MILESTONE_MULTI_USER_SERVER;
 271 
 272 
 273 /*
 274  * These services define the system being "up".  If none of them can come
 275  * online, then we will run sulogin on the console.  Note that the install ones
 276  * are for the miniroot and when installing CDs after the first.  can_come_up()
 277  * does the decision making, and an sulogin_thread() runs sulogin, which can be
 278  * started by dgraph_set_instance_state() or single_user_thread().
 279  *
 280  * NOTE: can_come_up() relies on SCF_MILESTONE_SINGLE_USER being the first
 281  * entry, which is only used when booting_to_single_user (boot -s) is set.
 282  * This is because when doing a "boot -s", sulogin is started from specials.c
 283  * after milestone/single-user comes online, for backwards compatibility.
 284  * In this case, SCF_MILESTONE_SINGLE_USER needs to be part of up_svcs
 285  * to ensure sulogin will be spawned if milestone/single-user cannot be reached.
 286  */
 287 static const char * const up_svcs[] = {
 288         SCF_MILESTONE_SINGLE_USER,
 289         CONSOLE_LOGIN_FMRI,
 290         "svc:/system/install-setup:default",
 291         "svc:/system/install:default",
 292         NULL
 293 };
 294 
 295 /* This array must have an element for each non-NULL element of up_svcs[]. */
 296 static graph_vertex_t *up_svcs_p[] = { NULL, NULL, NULL, NULL };
 297 
 298 /* These are for seed repository magic.  See can_come_up(). */
 299 static const char * const manifest_import = SCF_INSTANCE_MI;
 300 static graph_vertex_t *manifest_import_p = NULL;
 301 
 302 
 303 static char target_milestone_as_runlevel(void);
 304 static void graph_runlevel_changed(char rl, int online);
 305 static int dgraph_set_milestone(const char *, scf_handle_t *, boolean_t);
 306 static boolean_t should_be_in_subgraph(graph_vertex_t *v);
 307 static int mark_subtree(graph_edge_t *, void *);
 308 static boolean_t insubtree_dependents_down(graph_vertex_t *);
 309 
 310 /*
 311  * graph_vertex_compare()
 312  *      This function can compare either int *id or * graph_vertex_t *gv
 313  *      values, as the vertex id is always the first element of a
 314  *      graph_vertex structure.
 315  */
 316 /* ARGSUSED */
 317 static int
 318 graph_vertex_compare(const void *lc_arg, const void *rc_arg, void *private)
 319 {
 320         int lc_id = ((const graph_vertex_t *)lc_arg)->gv_id;
 321         int rc_id = *(int *)rc_arg;
 322 
 323         if (lc_id > rc_id)
 324                 return (1);
 325         if (lc_id < rc_id)
 326                 return (-1);
 327         return (0);
 328 }
 329 
 330 void
 331 graph_init()
 332 {
 333         graph_edge_pool = startd_list_pool_create("graph_edges",
 334             sizeof (graph_edge_t), offsetof(graph_edge_t, ge_link), NULL,
 335             UU_LIST_POOL_DEBUG);
 336         assert(graph_edge_pool != NULL);
 337 
 338         graph_vertex_pool = startd_list_pool_create("graph_vertices",
 339             sizeof (graph_vertex_t), offsetof(graph_vertex_t, gv_link),
 340             graph_vertex_compare, UU_LIST_POOL_DEBUG);
 341         assert(graph_vertex_pool != NULL);
 342 
 343         (void) pthread_mutex_init(&dgraph_lock, &mutex_attrs);
 344         (void) pthread_mutex_init(&single_user_thread_lock, &mutex_attrs);
 345         dgraph = startd_list_create(graph_vertex_pool, NULL, UU_LIST_SORTED);
 346         assert(dgraph != NULL);
 347 
 348         if (!st->st_initial)
 349                 current_runlevel = utmpx_get_runlevel();
 350 
 351         log_framework(LOG_DEBUG, "Initialized graph\n");
 352 }
 353 
 354 static graph_vertex_t *
 355 vertex_get_by_name(const char *name)
 356 {
 357         int id;
 358 
 359         assert(MUTEX_HELD(&dgraph_lock));
 360 
 361         id = dict_lookup_byname(name);
 362         if (id == -1)
 363                 return (NULL);
 364 
 365         return (uu_list_find(dgraph, &id, NULL, NULL));
 366 }
 367 
 368 static graph_vertex_t *
 369 vertex_get_by_id(int id)
 370 {
 371         assert(MUTEX_HELD(&dgraph_lock));
 372 
 373         if (id == -1)
 374                 return (NULL);
 375 
 376         return (uu_list_find(dgraph, &id, NULL, NULL));
 377 }
 378 
 379 /*
 380  * Creates a new vertex with the given name, adds it to the graph, and returns
 381  * a pointer to it.  The graph lock must be held by this thread on entry.
 382  */
 383 static graph_vertex_t *
 384 graph_add_vertex(const char *name)
 385 {
 386         int id;
 387         graph_vertex_t *v;
 388         void *p;
 389         uu_list_index_t idx;
 390 
 391         assert(MUTEX_HELD(&dgraph_lock));
 392 
 393         id = dict_insert(name);
 394 
 395         v = startd_zalloc(sizeof (*v));
 396 
 397         v->gv_id = id;
 398 
 399         v->gv_name = startd_alloc(strlen(name) + 1);
 400         (void) strcpy(v->gv_name, name);
 401 
 402         v->gv_dependencies = startd_list_create(graph_edge_pool, v, 0);
 403         v->gv_dependents = startd_list_create(graph_edge_pool, v, 0);
 404 
 405         p = uu_list_find(dgraph, &id, NULL, &idx);
 406         assert(p == NULL);
 407 
 408         uu_list_node_init(v, &v->gv_link, graph_vertex_pool);
 409         uu_list_insert(dgraph, v, idx);
 410 
 411         return (v);
 412 }
 413 
 414 /*
 415  * Removes v from the graph and frees it.  The graph should be locked by this
 416  * thread, and v should have no edges associated with it.
 417  */
 418 static void
 419 graph_remove_vertex(graph_vertex_t *v)
 420 {
 421         assert(MUTEX_HELD(&dgraph_lock));
 422 
 423         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 424         assert(uu_list_numnodes(v->gv_dependents) == 0);
 425         assert(v->gv_refs == 0);
 426 
 427         startd_free(v->gv_name, strlen(v->gv_name) + 1);
 428         uu_list_destroy(v->gv_dependencies);
 429         uu_list_destroy(v->gv_dependents);
 430         uu_list_remove(dgraph, v);
 431 
 432         startd_free(v, sizeof (graph_vertex_t));
 433 }
 434 
 435 static void
 436 graph_add_edge(graph_vertex_t *fv, graph_vertex_t *tv)
 437 {
 438         graph_edge_t *e, *re;
 439         int r;
 440 
 441         assert(MUTEX_HELD(&dgraph_lock));
 442 
 443         e = startd_alloc(sizeof (graph_edge_t));
 444         re = startd_alloc(sizeof (graph_edge_t));
 445 
 446         e->ge_parent = fv;
 447         e->ge_vertex = tv;
 448 
 449         re->ge_parent = tv;
 450         re->ge_vertex = fv;
 451 
 452         uu_list_node_init(e, &e->ge_link, graph_edge_pool);
 453         r = uu_list_insert_before(fv->gv_dependencies, NULL, e);
 454         assert(r == 0);
 455 
 456         uu_list_node_init(re, &re->ge_link, graph_edge_pool);
 457         r = uu_list_insert_before(tv->gv_dependents, NULL, re);
 458         assert(r == 0);
 459 }
 460 
 461 static void
 462 graph_remove_edge(graph_vertex_t *v, graph_vertex_t *dv)
 463 {
 464         graph_edge_t *e;
 465 
 466         for (e = uu_list_first(v->gv_dependencies);
 467             e != NULL;
 468             e = uu_list_next(v->gv_dependencies, e)) {
 469                 if (e->ge_vertex == dv) {
 470                         uu_list_remove(v->gv_dependencies, e);
 471                         startd_free(e, sizeof (graph_edge_t));
 472                         break;
 473                 }
 474         }
 475 
 476         for (e = uu_list_first(dv->gv_dependents);
 477             e != NULL;
 478             e = uu_list_next(dv->gv_dependents, e)) {
 479                 if (e->ge_vertex == v) {
 480                         uu_list_remove(dv->gv_dependents, e);
 481                         startd_free(e, sizeof (graph_edge_t));
 482                         break;
 483                 }
 484         }
 485 }
 486 
 487 static void
 488 remove_inst_vertex(graph_vertex_t *v)
 489 {
 490         graph_edge_t *e;
 491         graph_vertex_t *sv;
 492         int i;
 493 
 494         assert(MUTEX_HELD(&dgraph_lock));
 495         assert(uu_list_numnodes(v->gv_dependents) == 1);
 496         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 497         assert(v->gv_refs == 0);
 498         assert((v->gv_flags & GV_CONFIGURED) == 0);
 499 
 500         e = uu_list_first(v->gv_dependents);
 501         sv = e->ge_vertex;
 502         graph_remove_edge(sv, v);
 503 
 504         for (i = 0; up_svcs[i] != NULL; ++i) {
 505                 if (up_svcs_p[i] == v)
 506                         up_svcs_p[i] = NULL;
 507         }
 508 
 509         if (manifest_import_p == v)
 510                 manifest_import_p = NULL;
 511 
 512         graph_remove_vertex(v);
 513 
 514         if (uu_list_numnodes(sv->gv_dependencies) == 0 &&
 515             uu_list_numnodes(sv->gv_dependents) == 0 &&
 516             sv->gv_refs == 0)
 517                 graph_remove_vertex(sv);
 518 }
 519 
 520 static void
 521 graph_walk_dependents(graph_vertex_t *v, void (*func)(graph_vertex_t *, void *),
 522     void *arg)
 523 {
 524         graph_edge_t *e;
 525 
 526         for (e = uu_list_first(v->gv_dependents);
 527             e != NULL;
 528             e = uu_list_next(v->gv_dependents, e))
 529                 func(e->ge_vertex, arg);
 530 }
 531 
 532 static void
 533 graph_walk_dependencies(graph_vertex_t *v, void (*func)(graph_vertex_t *,
 534         void *), void *arg)
 535 {
 536         graph_edge_t *e;
 537 
 538         assert(MUTEX_HELD(&dgraph_lock));
 539 
 540         for (e = uu_list_first(v->gv_dependencies);
 541             e != NULL;
 542             e = uu_list_next(v->gv_dependencies, e)) {
 543 
 544                 func(e->ge_vertex, arg);
 545         }
 546 }
 547 
 548 /*
 549  * Generic graph walking function.
 550  *
 551  * Given a vertex, this function will walk either dependencies
 552  * (WALK_DEPENDENCIES) or dependents (WALK_DEPENDENTS) of a vertex recursively
 553  * for the entire graph.  It will avoid cycles and never visit the same vertex
 554  * twice.
 555  *
 556  * We avoid traversing exclusion dependencies, because they are allowed to
 557  * create cycles in the graph.  When propagating satisfiability, there is no
 558  * need to walk exclusion dependencies because exclude_all_satisfied() doesn't
 559  * test for satisfiability.
 560  *
 561  * The walker takes two callbacks.  The first is called before examining the
 562  * dependents of each vertex.  The second is called on each vertex after
 563  * examining its dependents.  This allows is_path_to() to construct a path only
 564  * after the target vertex has been found.
 565  */
 566 typedef enum {
 567         WALK_DEPENDENTS,
 568         WALK_DEPENDENCIES
 569 } graph_walk_dir_t;
 570 
 571 typedef int (*graph_walk_cb_t)(graph_vertex_t *, void *);
 572 
 573 typedef struct graph_walk_info {
 574         graph_walk_dir_t        gi_dir;
 575         uchar_t                 *gi_visited;    /* vertex bitmap */
 576         int                     (*gi_pre)(graph_vertex_t *, void *);
 577         void                    (*gi_post)(graph_vertex_t *, void *);
 578         void                    *gi_arg;        /* callback arg */
 579         int                     gi_ret;         /* return value */
 580 } graph_walk_info_t;
 581 
 582 static int
 583 graph_walk_recurse(graph_edge_t *e, graph_walk_info_t *gip)
 584 {
 585         uu_list_t *list;
 586         int r;
 587         graph_vertex_t *v = e->ge_vertex;
 588         int i;
 589         uint_t b;
 590 
 591         i = v->gv_id / 8;
 592         b = 1 << (v->gv_id % 8);
 593 
 594         /*
 595          * Check to see if we've visited this vertex already.
 596          */
 597         if (gip->gi_visited[i] & b)
 598                 return (UU_WALK_NEXT);
 599 
 600         gip->gi_visited[i] |= b;
 601 
 602         /*
 603          * Don't follow exclusions.
 604          */
 605         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
 606                 return (UU_WALK_NEXT);
 607 
 608         /*
 609          * Call pre-visit callback.  If this doesn't terminate the walk,
 610          * continue search.
 611          */
 612         if ((gip->gi_ret = gip->gi_pre(v, gip->gi_arg)) == UU_WALK_NEXT) {
 613                 /*
 614                  * Recurse using appropriate list.
 615                  */
 616                 if (gip->gi_dir == WALK_DEPENDENTS)
 617                         list = v->gv_dependents;
 618                 else
 619                         list = v->gv_dependencies;
 620 
 621                 r = uu_list_walk(list, (uu_walk_fn_t *)graph_walk_recurse,
 622                     gip, 0);
 623                 assert(r == 0);
 624         }
 625 
 626         /*
 627          * Callbacks must return either UU_WALK_NEXT or UU_WALK_DONE.
 628          */
 629         assert(gip->gi_ret == UU_WALK_NEXT || gip->gi_ret == UU_WALK_DONE);
 630 
 631         /*
 632          * If given a post-callback, call the function for every vertex.
 633          */
 634         if (gip->gi_post != NULL)
 635                 (void) gip->gi_post(v, gip->gi_arg);
 636 
 637         /*
 638          * Preserve the callback's return value.  If the callback returns
 639          * UU_WALK_DONE, then we propagate that to the caller in order to
 640          * terminate the walk.
 641          */
 642         return (gip->gi_ret);
 643 }
 644 
 645 static void
 646 graph_walk(graph_vertex_t *v, graph_walk_dir_t dir,
 647     int (*pre)(graph_vertex_t *, void *),
 648     void (*post)(graph_vertex_t *, void *), void *arg)
 649 {
 650         graph_walk_info_t gi;
 651         graph_edge_t fake;
 652         size_t sz = dictionary->dict_new_id / 8 + 1;
 653 
 654         gi.gi_visited = startd_zalloc(sz);
 655         gi.gi_pre = pre;
 656         gi.gi_post = post;
 657         gi.gi_arg = arg;
 658         gi.gi_dir = dir;
 659         gi.gi_ret = 0;
 660 
 661         /*
 662          * Fake up an edge for the first iteration
 663          */
 664         fake.ge_vertex = v;
 665         (void) graph_walk_recurse(&fake, &gi);
 666 
 667         startd_free(gi.gi_visited, sz);
 668 }
 669 
 670 typedef struct child_search {
 671         int     id;             /* id of vertex to look for */
 672         uint_t  depth;          /* recursion depth */
 673         /*
 674          * While the vertex is not found, path is NULL.  After the search, if
 675          * the vertex was found then path should point to a -1-terminated
 676          * array of vertex id's which constitute the path to the vertex.
 677          */
 678         int     *path;
 679 } child_search_t;
 680 
 681 static int
 682 child_pre(graph_vertex_t *v, void *arg)
 683 {
 684         child_search_t *cs = arg;
 685 
 686         cs->depth++;
 687 
 688         if (v->gv_id == cs->id) {
 689                 cs->path = startd_alloc((cs->depth + 1) * sizeof (int));
 690                 cs->path[cs->depth] = -1;
 691                 return (UU_WALK_DONE);
 692         }
 693 
 694         return (UU_WALK_NEXT);
 695 }
 696 
 697 static void
 698 child_post(graph_vertex_t *v, void *arg)
 699 {
 700         child_search_t *cs = arg;
 701 
 702         cs->depth--;
 703 
 704         if (cs->path != NULL)
 705                 cs->path[cs->depth] = v->gv_id;
 706 }
 707 
 708 /*
 709  * Look for a path from from to to.  If one exists, returns a pointer to
 710  * a NULL-terminated array of pointers to the vertices along the path.  If
 711  * there is no path, returns NULL.
 712  */
 713 static int *
 714 is_path_to(graph_vertex_t *from, graph_vertex_t *to)
 715 {
 716         child_search_t cs;
 717 
 718         cs.id = to->gv_id;
 719         cs.depth = 0;
 720         cs.path = NULL;
 721 
 722         graph_walk(from, WALK_DEPENDENCIES, child_pre, child_post, &cs);
 723 
 724         return (cs.path);
 725 }
 726 
 727 /*
 728  * Given an array of int's as returned by is_path_to, allocates a string of
 729  * their names joined by newlines.  Returns the size of the allocated buffer
 730  * in *sz and frees path.
 731  */
 732 static void
 733 path_to_str(int *path, char **cpp, size_t *sz)
 734 {
 735         int i;
 736         graph_vertex_t *v;
 737         size_t allocd, new_allocd;
 738         char *new, *name;
 739 
 740         assert(MUTEX_HELD(&dgraph_lock));
 741         assert(path[0] != -1);
 742 
 743         allocd = 1;
 744         *cpp = startd_alloc(1);
 745         (*cpp)[0] = '\0';
 746 
 747         for (i = 0; path[i] != -1; ++i) {
 748                 name = NULL;
 749 
 750                 v = vertex_get_by_id(path[i]);
 751 
 752                 if (v == NULL)
 753                         name = "<deleted>";
 754                 else if (v->gv_type == GVT_INST || v->gv_type == GVT_SVC)
 755                         name = v->gv_name;
 756 
 757                 if (name != NULL) {
 758                         new_allocd = allocd + strlen(name) + 1;
 759                         new = startd_alloc(new_allocd);
 760                         (void) strcpy(new, *cpp);
 761                         (void) strcat(new, name);
 762                         (void) strcat(new, "\n");
 763 
 764                         startd_free(*cpp, allocd);
 765 
 766                         *cpp = new;
 767                         allocd = new_allocd;
 768                 }
 769         }
 770 
 771         startd_free(path, sizeof (int) * (i + 1));
 772 
 773         *sz = allocd;
 774 }
 775 
 776 
 777 /*
 778  * This function along with run_sulogin() implements an exclusion relationship
 779  * between system/console-login and sulogin.  run_sulogin() will fail if
 780  * system/console-login is online, and the graph engine should call
 781  * graph_clogin_start() to bring system/console-login online, which defers the
 782  * start if sulogin is running.
 783  */
 784 static void
 785 graph_clogin_start(graph_vertex_t *v)
 786 {
 787         assert(MUTEX_HELD(&dgraph_lock));
 788 
 789         if (sulogin_running)
 790                 console_login_ready = B_TRUE;
 791         else
 792                 vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 793 }
 794 
 795 static void
 796 graph_su_start(graph_vertex_t *v)
 797 {
 798         /*
 799          * /etc/inittab used to have the initial /sbin/rcS as a 'sysinit'
 800          * entry with a runlevel of 'S', before jumping to the final
 801          * target runlevel (as set in initdefault).  We mimic that legacy
 802          * behavior here.
 803          */
 804         utmpx_set_runlevel('S', '0', B_FALSE);
 805         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 806 }
 807 
 808 static void
 809 graph_post_su_online(void)
 810 {
 811         graph_runlevel_changed('S', 1);
 812 }
 813 
 814 static void
 815 graph_post_su_disable(void)
 816 {
 817         graph_runlevel_changed('S', 0);
 818 }
 819 
 820 static void
 821 graph_post_mu_online(void)
 822 {
 823         graph_runlevel_changed('2', 1);
 824 }
 825 
 826 static void
 827 graph_post_mu_disable(void)
 828 {
 829         graph_runlevel_changed('2', 0);
 830 }
 831 
 832 static void
 833 graph_post_mus_online(void)
 834 {
 835         graph_runlevel_changed('3', 1);
 836 }
 837 
 838 static void
 839 graph_post_mus_disable(void)
 840 {
 841         graph_runlevel_changed('3', 0);
 842 }
 843 
 844 static struct special_vertex_info {
 845         const char      *name;
 846         void            (*start_f)(graph_vertex_t *);
 847         void            (*post_online_f)(void);
 848         void            (*post_disable_f)(void);
 849 } special_vertices[] = {
 850         { CONSOLE_LOGIN_FMRI, graph_clogin_start, NULL, NULL },
 851         { SCF_MILESTONE_SINGLE_USER, graph_su_start,
 852             graph_post_su_online, graph_post_su_disable },
 853         { SCF_MILESTONE_MULTI_USER, NULL,
 854             graph_post_mu_online, graph_post_mu_disable },
 855         { SCF_MILESTONE_MULTI_USER_SERVER, NULL,
 856             graph_post_mus_online, graph_post_mus_disable },
 857         { NULL },
 858 };
 859 
 860 
 861 void
 862 vertex_send_event(graph_vertex_t *v, restarter_event_type_t e)
 863 {
 864         switch (e) {
 865         case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
 866                 assert(v->gv_state == RESTARTER_STATE_UNINIT);
 867 
 868                 MUTEX_LOCK(&st->st_load_lock);
 869                 st->st_load_instances++;
 870                 MUTEX_UNLOCK(&st->st_load_lock);
 871                 break;
 872 
 873         case RESTARTER_EVENT_TYPE_ENABLE:
 874                 log_framework(LOG_DEBUG, "Enabling %s.\n", v->gv_name);
 875                 assert(v->gv_state == RESTARTER_STATE_UNINIT ||
 876                     v->gv_state == RESTARTER_STATE_DISABLED ||
 877                     v->gv_state == RESTARTER_STATE_MAINT);
 878                 break;
 879 
 880         case RESTARTER_EVENT_TYPE_DISABLE:
 881         case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
 882                 log_framework(LOG_DEBUG, "Disabling %s.\n", v->gv_name);
 883                 assert(v->gv_state != RESTARTER_STATE_DISABLED);
 884                 break;
 885 
 886         case RESTARTER_EVENT_TYPE_STOP_RESET:
 887         case RESTARTER_EVENT_TYPE_STOP:
 888                 log_framework(LOG_DEBUG, "Stopping %s.\n", v->gv_name);
 889                 assert(v->gv_state == RESTARTER_STATE_DEGRADED ||
 890                     v->gv_state == RESTARTER_STATE_ONLINE);
 891                 break;
 892 
 893         case RESTARTER_EVENT_TYPE_START:
 894                 log_framework(LOG_DEBUG, "Starting %s.\n", v->gv_name);
 895                 assert(v->gv_state == RESTARTER_STATE_OFFLINE);
 896                 break;
 897 
 898         case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
 899         case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
 900         case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
 901         case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
 902         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
 903         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
 904         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
 905         case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
 906         case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
 907                 break;
 908 
 909         default:
 910 #ifndef NDEBUG
 911                 uu_warn("%s:%d: Bad event %d.\n", __FILE__, __LINE__, e);
 912 #endif
 913                 abort();
 914         }
 915 
 916         restarter_protocol_send_event(v->gv_name, v->gv_restarter_channel, e,
 917             v->gv_reason);
 918 }
 919 
 920 static void
 921 graph_unset_restarter(graph_vertex_t *v)
 922 {
 923         assert(MUTEX_HELD(&dgraph_lock));
 924         assert(v->gv_flags & GV_CONFIGURED);
 925 
 926         vertex_send_event(v, RESTARTER_EVENT_TYPE_REMOVE_INSTANCE);
 927 
 928         if (v->gv_restarter_id != -1) {
 929                 graph_vertex_t *rv;
 930 
 931                 rv = vertex_get_by_id(v->gv_restarter_id);
 932                 graph_remove_edge(v, rv);
 933         }
 934 
 935         v->gv_restarter_id = -1;
 936         v->gv_restarter_channel = NULL;
 937 }
 938 
 939 /*
 940  * Return VERTEX_REMOVED when the vertex passed in argument is deleted from the
 941  * dgraph otherwise return VERTEX_INUSE.
 942  */
 943 static int
 944 free_if_unrefed(graph_vertex_t *v)
 945 {
 946         assert(MUTEX_HELD(&dgraph_lock));
 947 
 948         if (v->gv_refs > 0)
 949                 return (VERTEX_INUSE);
 950 
 951         if (v->gv_type == GVT_SVC &&
 952             uu_list_numnodes(v->gv_dependents) == 0 &&
 953             uu_list_numnodes(v->gv_dependencies) == 0) {
 954                 graph_remove_vertex(v);
 955                 return (VERTEX_REMOVED);
 956         } else if (v->gv_type == GVT_INST &&
 957             (v->gv_flags & GV_CONFIGURED) == 0 &&
 958             uu_list_numnodes(v->gv_dependents) == 1 &&
 959             uu_list_numnodes(v->gv_dependencies) == 0) {
 960                 remove_inst_vertex(v);
 961                 return (VERTEX_REMOVED);
 962         }
 963 
 964         return (VERTEX_INUSE);
 965 }
 966 
 967 static void
 968 delete_depgroup(graph_vertex_t *v)
 969 {
 970         graph_edge_t *e;
 971         graph_vertex_t *dv;
 972 
 973         assert(MUTEX_HELD(&dgraph_lock));
 974         assert(v->gv_type == GVT_GROUP);
 975         assert(uu_list_numnodes(v->gv_dependents) == 0);
 976 
 977         while ((e = uu_list_first(v->gv_dependencies)) != NULL) {
 978                 dv = e->ge_vertex;
 979 
 980                 graph_remove_edge(v, dv);
 981 
 982                 switch (dv->gv_type) {
 983                 case GVT_INST:          /* instance dependency */
 984                 case GVT_SVC:           /* service dependency */
 985                         (void) free_if_unrefed(dv);
 986                         break;
 987 
 988                 case GVT_FILE:          /* file dependency */
 989                         assert(uu_list_numnodes(dv->gv_dependencies) == 0);
 990                         if (uu_list_numnodes(dv->gv_dependents) == 0)
 991                                 graph_remove_vertex(dv);
 992                         break;
 993 
 994                 default:
 995 #ifndef NDEBUG
 996                         uu_warn("%s:%d: Unexpected node type %d", __FILE__,
 997                             __LINE__, dv->gv_type);
 998 #endif
 999                         abort();
1000                 }
1001         }
1002 
1003         graph_remove_vertex(v);
1004 }
1005 
1006 static int
1007 delete_instance_deps_cb(graph_edge_t *e, void **ptrs)
1008 {
1009         graph_vertex_t *v = ptrs[0];
1010         boolean_t delete_restarter_dep = (boolean_t)ptrs[1];
1011         graph_vertex_t *dv;
1012 
1013         dv = e->ge_vertex;
1014 
1015         /*
1016          * We have four possibilities here:
1017          *   - GVT_INST: restarter
1018          *   - GVT_GROUP - GVT_INST: instance dependency
1019          *   - GVT_GROUP - GVT_SVC - GV_INST: service dependency
1020          *   - GVT_GROUP - GVT_FILE: file dependency
1021          */
1022         switch (dv->gv_type) {
1023         case GVT_INST:  /* restarter */
1024                 assert(dv->gv_id == v->gv_restarter_id);
1025                 if (delete_restarter_dep)
1026                         graph_remove_edge(v, dv);
1027                 break;
1028 
1029         case GVT_GROUP: /* pg dependency */
1030                 graph_remove_edge(v, dv);
1031                 delete_depgroup(dv);
1032                 break;
1033 
1034         case GVT_FILE:
1035                 /* These are currently not direct dependencies */
1036 
1037         default:
1038 #ifndef NDEBUG
1039                 uu_warn("%s:%d: Bad vertex type %d.\n", __FILE__, __LINE__,
1040                     dv->gv_type);
1041 #endif
1042                 abort();
1043         }
1044 
1045         return (UU_WALK_NEXT);
1046 }
1047 
1048 static void
1049 delete_instance_dependencies(graph_vertex_t *v, boolean_t delete_restarter_dep)
1050 {
1051         void *ptrs[2];
1052         int r;
1053 
1054         assert(MUTEX_HELD(&dgraph_lock));
1055         assert(v->gv_type == GVT_INST);
1056 
1057         ptrs[0] = v;
1058         ptrs[1] = (void *)delete_restarter_dep;
1059 
1060         r = uu_list_walk(v->gv_dependencies,
1061             (uu_walk_fn_t *)delete_instance_deps_cb, &ptrs, UU_WALK_ROBUST);
1062         assert(r == 0);
1063 }
1064 
1065 /*
1066  * int graph_insert_vertex_unconfigured()
1067  *   Insert a vertex without sending any restarter events. If the vertex
1068  *   already exists or creation is successful, return a pointer to it in *vp.
1069  *
1070  *   If type is not GVT_GROUP, dt can remain unset.
1071  *
1072  *   Returns 0, EEXIST, or EINVAL if the arguments are invalid (i.e., fmri
1073  *   doesn't agree with type, or type doesn't agree with dt).
1074  */
1075 static int
1076 graph_insert_vertex_unconfigured(const char *fmri, gv_type_t type,
1077     depgroup_type_t dt, restarter_error_t rt, graph_vertex_t **vp)
1078 {
1079         int r;
1080         int i;
1081 
1082         assert(MUTEX_HELD(&dgraph_lock));
1083 
1084         switch (type) {
1085         case GVT_SVC:
1086         case GVT_INST:
1087                 if (strncmp(fmri, "svc:", sizeof ("svc:") - 1) != 0)
1088                         return (EINVAL);
1089                 break;
1090 
1091         case GVT_FILE:
1092                 if (strncmp(fmri, "file:", sizeof ("file:") - 1) != 0)
1093                         return (EINVAL);
1094                 break;
1095 
1096         case GVT_GROUP:
1097                 if (dt <= 0 || rt < 0)
1098                         return (EINVAL);
1099                 break;
1100 
1101         default:
1102 #ifndef NDEBUG
1103                 uu_warn("%s:%d: Unknown type %d.\n", __FILE__, __LINE__, type);
1104 #endif
1105                 abort();
1106         }
1107 
1108         *vp = vertex_get_by_name(fmri);
1109         if (*vp != NULL)
1110                 return (EEXIST);
1111 
1112         *vp = graph_add_vertex(fmri);
1113 
1114         (*vp)->gv_type = type;
1115         (*vp)->gv_depgroup = dt;
1116         (*vp)->gv_restart = rt;
1117 
1118         (*vp)->gv_flags = 0;
1119         (*vp)->gv_state = RESTARTER_STATE_NONE;
1120 
1121         for (i = 0; special_vertices[i].name != NULL; ++i) {
1122                 if (strcmp(fmri, special_vertices[i].name) == 0) {
1123                         (*vp)->gv_start_f = special_vertices[i].start_f;
1124                         (*vp)->gv_post_online_f =
1125                             special_vertices[i].post_online_f;
1126                         (*vp)->gv_post_disable_f =
1127                             special_vertices[i].post_disable_f;
1128                         break;
1129                 }
1130         }
1131 
1132         (*vp)->gv_restarter_id = -1;
1133         (*vp)->gv_restarter_channel = 0;
1134 
1135         if (type == GVT_INST) {
1136                 char *sfmri;
1137                 graph_vertex_t *sv;
1138 
1139                 sfmri = inst_fmri_to_svc_fmri(fmri);
1140                 sv = vertex_get_by_name(sfmri);
1141                 if (sv == NULL) {
1142                         r = graph_insert_vertex_unconfigured(sfmri, GVT_SVC, 0,
1143                             0, &sv);
1144                         assert(r == 0);
1145                 }
1146                 startd_free(sfmri, max_scf_fmri_size);
1147 
1148                 graph_add_edge(sv, *vp);
1149         }
1150 
1151         /*
1152          * If this vertex is in the subgraph, mark it as so, for both
1153          * GVT_INST and GVT_SERVICE verteces.
1154          * A GVT_SERVICE vertex can only be in the subgraph if another instance
1155          * depends on it, in which case it's already been added to the graph
1156          * and marked as in the subgraph (by refresh_vertex()).  If a
1157          * GVT_SERVICE vertex was freshly added (by the code above), it means
1158          * that it has no dependents, and cannot be in the subgraph.
1159          * Regardless of this, we still check that gv_flags includes
1160          * GV_INSUBGRAPH in the event that future behavior causes the above
1161          * code to add a GVT_SERVICE vertex which should be in the subgraph.
1162          */
1163 
1164         (*vp)->gv_flags |= (should_be_in_subgraph(*vp)? GV_INSUBGRAPH : 0);
1165 
1166         return (0);
1167 }
1168 
1169 /*
1170  * Returns 0 on success or ELOOP if the dependency would create a cycle.
1171  */
1172 static int
1173 graph_insert_dependency(graph_vertex_t *fv, graph_vertex_t *tv, int **pathp)
1174 {
1175         hrtime_t now;
1176 
1177         assert(MUTEX_HELD(&dgraph_lock));
1178 
1179         /* cycle detection */
1180         now = gethrtime();
1181 
1182         /* Don't follow exclusions. */
1183         if (!(fv->gv_type == GVT_GROUP &&
1184             fv->gv_depgroup == DEPGRP_EXCLUDE_ALL)) {
1185                 *pathp = is_path_to(tv, fv);
1186                 if (*pathp)
1187                         return (ELOOP);
1188         }
1189 
1190         dep_cycle_ns += gethrtime() - now;
1191         ++dep_inserts;
1192         now = gethrtime();
1193 
1194         graph_add_edge(fv, tv);
1195 
1196         dep_insert_ns += gethrtime() - now;
1197 
1198         /* Check if the dependency adds the "to" vertex to the subgraph */
1199         tv->gv_flags |= (should_be_in_subgraph(tv) ? GV_INSUBGRAPH : 0);
1200 
1201         return (0);
1202 }
1203 
1204 static int
1205 inst_running(graph_vertex_t *v)
1206 {
1207         assert(v->gv_type == GVT_INST);
1208 
1209         if (v->gv_state == RESTARTER_STATE_ONLINE ||
1210             v->gv_state == RESTARTER_STATE_DEGRADED)
1211                 return (1);
1212 
1213         return (0);
1214 }
1215 
1216 /*
1217  * The dependency evaluation functions return
1218  *   1 - dependency satisfied
1219  *   0 - dependency unsatisfied
1220  *   -1 - dependency unsatisfiable (without administrator intervention)
1221  *
1222  * The functions also take a boolean satbility argument.  When true, the
1223  * functions may recurse in order to determine satisfiability.
1224  */
1225 static int require_any_satisfied(graph_vertex_t *, boolean_t);
1226 static int dependency_satisfied(graph_vertex_t *, boolean_t);
1227 
1228 /*
1229  * A require_all dependency is unsatisfied if any elements are unsatisfied.  It
1230  * is unsatisfiable if any elements are unsatisfiable.
1231  */
1232 static int
1233 require_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1234 {
1235         graph_edge_t *edge;
1236         int i;
1237         boolean_t any_unsatisfied;
1238 
1239         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1240                 return (1);
1241 
1242         any_unsatisfied = B_FALSE;
1243 
1244         for (edge = uu_list_first(groupv->gv_dependencies);
1245             edge != NULL;
1246             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1247                 i = dependency_satisfied(edge->ge_vertex, satbility);
1248                 if (i == 1)
1249                         continue;
1250 
1251                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1252                     "require_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1253                     edge->ge_vertex->gv_name, i == 0 ? "ed" : "able");
1254 
1255                 if (!satbility)
1256                         return (0);
1257 
1258                 if (i == -1)
1259                         return (-1);
1260 
1261                 any_unsatisfied = B_TRUE;
1262         }
1263 
1264         return (any_unsatisfied ? 0 : 1);
1265 }
1266 
1267 /*
1268  * A require_any dependency is satisfied if any element is satisfied.  It is
1269  * satisfiable if any element is satisfiable.
1270  */
1271 static int
1272 require_any_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1273 {
1274         graph_edge_t *edge;
1275         int s;
1276         boolean_t satisfiable;
1277 
1278         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1279                 return (1);
1280 
1281         satisfiable = B_FALSE;
1282 
1283         for (edge = uu_list_first(groupv->gv_dependencies);
1284             edge != NULL;
1285             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1286                 s = dependency_satisfied(edge->ge_vertex, satbility);
1287 
1288                 if (s == 1)
1289                         return (1);
1290 
1291                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1292                     "require_any(%s): %s is unsatisfi%s.\n",
1293                     groupv->gv_name, edge->ge_vertex->gv_name,
1294                     s == 0 ? "ed" : "able");
1295 
1296                 if (satbility && s == 0)
1297                         satisfiable = B_TRUE;
1298         }
1299 
1300         return (!satbility || satisfiable ? 0 : -1);
1301 }
1302 
1303 /*
1304  * An optional_all dependency only considers elements which are configured,
1305  * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1306  * is unsatisfied.
1307  *
1308  * Offline dependencies which are waiting for a dependency to come online are
1309  * unsatisfied.  Offline dependences which cannot possibly come online
1310  * (unsatisfiable) are always considered satisfied.
1311  */
1312 static int
1313 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1314 {
1315         graph_edge_t *edge;
1316         graph_vertex_t *v;
1317         boolean_t any_qualified;
1318         boolean_t any_unsatisfied;
1319         int i;
1320 
1321         any_qualified = B_FALSE;
1322         any_unsatisfied = B_FALSE;
1323 
1324         for (edge = uu_list_first(groupv->gv_dependencies);
1325             edge != NULL;
1326             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1327                 v = edge->ge_vertex;
1328 
1329                 switch (v->gv_type) {
1330                 case GVT_INST:
1331                         /* Skip missing or disabled instances */
1332                         if ((v->gv_flags & (GV_CONFIGURED | GV_ENABLED)) !=
1333                             (GV_CONFIGURED | GV_ENABLED))
1334                                 continue;
1335 
1336                         if (v->gv_state == RESTARTER_STATE_MAINT)
1337                                 continue;
1338 
1339                         if (v->gv_flags & GV_TOOFFLINE)
1340                                 continue;
1341 
1342                         any_qualified = B_TRUE;
1343                         if (v->gv_state == RESTARTER_STATE_OFFLINE) {
1344                                 /*
1345                                  * For offline dependencies, treat unsatisfiable
1346                                  * as satisfied.
1347                                  */
1348                                 i = dependency_satisfied(v, B_TRUE);
1349                                 if (i == -1)
1350                                         i = 1;
1351                         } else if (v->gv_state == RESTARTER_STATE_DISABLED) {
1352                                 /*
1353                                  * The service is enabled, but hasn't
1354                                  * transitioned out of disabled yet.  Treat it
1355                                  * as unsatisfied (not unsatisfiable).
1356                                  */
1357                                 i = 0;
1358                         } else {
1359                                 i = dependency_satisfied(v, satbility);
1360                         }
1361                         break;
1362 
1363                 case GVT_FILE:
1364                         any_qualified = B_TRUE;
1365                         i = dependency_satisfied(v, satbility);
1366 
1367                         break;
1368 
1369                 case GVT_SVC: {
1370                         boolean_t svc_any_qualified;
1371                         boolean_t svc_satisfied;
1372                         boolean_t svc_satisfiable;
1373                         graph_vertex_t *v2;
1374                         graph_edge_t *e2;
1375 
1376                         svc_any_qualified = B_FALSE;
1377                         svc_satisfied = B_FALSE;
1378                         svc_satisfiable = B_FALSE;
1379 
1380                         for (e2 = uu_list_first(v->gv_dependencies);
1381                             e2 != NULL;
1382                             e2 = uu_list_next(v->gv_dependencies, e2)) {
1383                                 v2 = e2->ge_vertex;
1384                                 assert(v2->gv_type == GVT_INST);
1385 
1386                                 if ((v2->gv_flags &
1387                                     (GV_CONFIGURED | GV_ENABLED)) !=
1388                                     (GV_CONFIGURED | GV_ENABLED))
1389                                         continue;
1390 
1391                                 if (v2->gv_state == RESTARTER_STATE_MAINT)
1392                                         continue;
1393 
1394                                 if (v2->gv_flags & GV_TOOFFLINE)
1395                                         continue;
1396 
1397                                 svc_any_qualified = B_TRUE;
1398 
1399                                 if (v2->gv_state == RESTARTER_STATE_OFFLINE) {
1400                                         /*
1401                                          * For offline dependencies, treat
1402                                          * unsatisfiable as satisfied.
1403                                          */
1404                                         i = dependency_satisfied(v2, B_TRUE);
1405                                         if (i == -1)
1406                                                 i = 1;
1407                                 } else if (v2->gv_state ==
1408                                     RESTARTER_STATE_DISABLED) {
1409                                         i = 0;
1410                                 } else {
1411                                         i = dependency_satisfied(v2, satbility);
1412                                 }
1413 
1414                                 if (i == 1) {
1415                                         svc_satisfied = B_TRUE;
1416                                         break;
1417                                 }
1418                                 if (i == 0)
1419                                         svc_satisfiable = B_TRUE;
1420                         }
1421 
1422                         if (!svc_any_qualified)
1423                                 continue;
1424                         any_qualified = B_TRUE;
1425                         if (svc_satisfied) {
1426                                 i = 1;
1427                         } else if (svc_satisfiable) {
1428                                 i = 0;
1429                         } else {
1430                                 i = -1;
1431                         }
1432                         break;
1433                 }
1434 
1435                 case GVT_GROUP:
1436                 default:
1437 #ifndef NDEBUG
1438                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1439                             __LINE__, v->gv_type);
1440 #endif
1441                         abort();
1442                 }
1443 
1444                 if (i == 1)
1445                         continue;
1446 
1447                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1448                     "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1449                     v->gv_name, i == 0 ? "ed" : "able");
1450 
1451                 if (!satbility)
1452                         return (0);
1453                 if (i == -1)
1454                         return (-1);
1455                 any_unsatisfied = B_TRUE;
1456         }
1457 
1458         if (!any_qualified)
1459                 return (1);
1460 
1461         return (any_unsatisfied ? 0 : 1);
1462 }
1463 
1464 /*
1465  * An exclude_all dependency is unsatisfied if any non-service element is
1466  * satisfied or any service instance which is configured, enabled, and not in
1467  * maintenance is satisfied.  Usually when unsatisfied, it is also
1468  * unsatisfiable.
1469  */
1470 #define LOG_EXCLUDE(u, v)                                               \
1471         log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,                   \
1472             "exclude_all(%s): %s is satisfied.\n",                      \
1473             (u)->gv_name, (v)->gv_name)
1474 
1475 /* ARGSUSED */
1476 static int
1477 exclude_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1478 {
1479         graph_edge_t *edge, *e2;
1480         graph_vertex_t *v, *v2;
1481 
1482         for (edge = uu_list_first(groupv->gv_dependencies);
1483             edge != NULL;
1484             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1485                 v = edge->ge_vertex;
1486 
1487                 switch (v->gv_type) {
1488                 case GVT_INST:
1489                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1490                                 continue;
1491 
1492                         switch (v->gv_state) {
1493                         case RESTARTER_STATE_ONLINE:
1494                         case RESTARTER_STATE_DEGRADED:
1495                                 LOG_EXCLUDE(groupv, v);
1496                                 return (v->gv_flags & GV_ENABLED ? -1 : 0);
1497 
1498                         case RESTARTER_STATE_OFFLINE:
1499                         case RESTARTER_STATE_UNINIT:
1500                                 LOG_EXCLUDE(groupv, v);
1501                                 return (0);
1502 
1503                         case RESTARTER_STATE_DISABLED:
1504                         case RESTARTER_STATE_MAINT:
1505                                 continue;
1506 
1507                         default:
1508 #ifndef NDEBUG
1509                                 uu_warn("%s:%d: Unexpected vertex state %d.\n",
1510                                     __FILE__, __LINE__, v->gv_state);
1511 #endif
1512                                 abort();
1513                         }
1514                         /* NOTREACHED */
1515 
1516                 case GVT_SVC:
1517                         break;
1518 
1519                 case GVT_FILE:
1520                         if (!file_ready(v))
1521                                 continue;
1522                         LOG_EXCLUDE(groupv, v);
1523                         return (-1);
1524 
1525                 case GVT_GROUP:
1526                 default:
1527 #ifndef NDEBUG
1528                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1529                             __LINE__, v->gv_type);
1530 #endif
1531                         abort();
1532                 }
1533 
1534                 /* v represents a service */
1535                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1536                         continue;
1537 
1538                 for (e2 = uu_list_first(v->gv_dependencies);
1539                     e2 != NULL;
1540                     e2 = uu_list_next(v->gv_dependencies, e2)) {
1541                         v2 = e2->ge_vertex;
1542                         assert(v2->gv_type == GVT_INST);
1543 
1544                         if ((v2->gv_flags & GV_CONFIGURED) == 0)
1545                                 continue;
1546 
1547                         switch (v2->gv_state) {
1548                         case RESTARTER_STATE_ONLINE:
1549                         case RESTARTER_STATE_DEGRADED:
1550                                 LOG_EXCLUDE(groupv, v2);
1551                                 return (v2->gv_flags & GV_ENABLED ? -1 : 0);
1552 
1553                         case RESTARTER_STATE_OFFLINE:
1554                         case RESTARTER_STATE_UNINIT:
1555                                 LOG_EXCLUDE(groupv, v2);
1556                                 return (0);
1557 
1558                         case RESTARTER_STATE_DISABLED:
1559                         case RESTARTER_STATE_MAINT:
1560                                 continue;
1561 
1562                         default:
1563 #ifndef NDEBUG
1564                                 uu_warn("%s:%d: Unexpected vertex type %d.\n",
1565                                     __FILE__, __LINE__, v2->gv_type);
1566 #endif
1567                                 abort();
1568                         }
1569                 }
1570         }
1571 
1572         return (1);
1573 }
1574 
1575 /*
1576  * int instance_satisfied()
1577  *   Determine if all the dependencies are satisfied for the supplied instance
1578  *   vertex. Return 1 if they are, 0 if they aren't, and -1 if they won't be
1579  *   without administrator intervention.
1580  */
1581 static int
1582 instance_satisfied(graph_vertex_t *v, boolean_t satbility)
1583 {
1584         assert(v->gv_type == GVT_INST);
1585         assert(!inst_running(v));
1586 
1587         return (require_all_satisfied(v, satbility));
1588 }
1589 
1590 /*
1591  * Decide whether v can satisfy a dependency.  v can either be a child of
1592  * a group vertex, or of an instance vertex.
1593  */
1594 static int
1595 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1596 {
1597         switch (v->gv_type) {
1598         case GVT_INST:
1599                 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1600                         if (v->gv_flags & GV_DEATHROW) {
1601                                 /*
1602                                  * A dependency on an instance with GV_DEATHROW
1603                                  * flag is always considered as satisfied.
1604                                  */
1605                                 return (1);
1606                         }
1607                         return (-1);
1608                 }
1609 
1610                 /*
1611                  * Any vertex with the GV_TOOFFLINE flag set is guaranteed
1612                  * to have its dependencies unsatisfiable.
1613                  */
1614                 if (v->gv_flags & GV_TOOFFLINE)
1615                         return (-1);
1616 
1617                 switch (v->gv_state) {
1618                 case RESTARTER_STATE_ONLINE:
1619                 case RESTARTER_STATE_DEGRADED:
1620                         return (1);
1621 
1622                 case RESTARTER_STATE_OFFLINE:
1623                         if (!satbility)
1624                                 return (0);
1625                         return (instance_satisfied(v, satbility) != -1 ?
1626                             0 : -1);
1627 
1628                 case RESTARTER_STATE_DISABLED:
1629                 case RESTARTER_STATE_MAINT:
1630                         return (-1);
1631 
1632                 case RESTARTER_STATE_UNINIT:
1633                         return (0);
1634 
1635                 default:
1636 #ifndef NDEBUG
1637                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
1638                             __FILE__, __LINE__, v->gv_state);
1639 #endif
1640                         abort();
1641                         /* NOTREACHED */
1642                 }
1643 
1644         case GVT_SVC:
1645                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1646                         return (-1);
1647                 return (require_any_satisfied(v, satbility));
1648 
1649         case GVT_FILE:
1650                 /* i.e., we assume files will not be automatically generated */
1651                 return (file_ready(v) ? 1 : -1);
1652 
1653         case GVT_GROUP:
1654                 break;
1655 
1656         default:
1657 #ifndef NDEBUG
1658                 uu_warn("%s:%d: Unexpected node type %d.\n", __FILE__, __LINE__,
1659                     v->gv_type);
1660 #endif
1661                 abort();
1662                 /* NOTREACHED */
1663         }
1664 
1665         switch (v->gv_depgroup) {
1666         case DEPGRP_REQUIRE_ANY:
1667                 return (require_any_satisfied(v, satbility));
1668 
1669         case DEPGRP_REQUIRE_ALL:
1670                 return (require_all_satisfied(v, satbility));
1671 
1672         case DEPGRP_OPTIONAL_ALL:
1673                 return (optional_all_satisfied(v, satbility));
1674 
1675         case DEPGRP_EXCLUDE_ALL:
1676                 return (exclude_all_satisfied(v, satbility));
1677 
1678         default:
1679 #ifndef NDEBUG
1680                 uu_warn("%s:%d: Unknown dependency grouping %d.\n", __FILE__,
1681                     __LINE__, v->gv_depgroup);
1682 #endif
1683                 abort();
1684         }
1685 }
1686 
1687 void
1688 graph_start_if_satisfied(graph_vertex_t *v)
1689 {
1690         if (v->gv_state == RESTARTER_STATE_OFFLINE &&
1691             instance_satisfied(v, B_FALSE) == 1) {
1692                 if (v->gv_start_f == NULL)
1693                         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
1694                 else
1695                         v->gv_start_f(v);
1696         }
1697 }
1698 
1699 /*
1700  * propagate_satbility()
1701  *
1702  * This function is used when the given vertex changes state in such a way that
1703  * one of its dependents may become unsatisfiable.  This happens when an
1704  * instance transitions between offline -> online, or from !running ->
1705  * maintenance, as well as when an instance is removed from the graph.
1706  *
1707  * We have to walk all the dependents, since optional_all dependencies several
1708  * levels up could become (un)satisfied, instead of unsatisfiable.  For example,
1709  *
1710  *      +-----+  optional_all  +-----+  require_all  +-----+
1711  *      |  A  |--------------->|  B  |-------------->|  C  |
1712  *      +-----+                +-----+               +-----+
1713  *
1714  *                                              offline -> maintenance
1715  *
1716  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1717  * an optional dependency, what was previously an unsatisfiable situation is now
1718  * satisfied (B will never come online, even though its state hasn't changed).
1719  *
1720  * Note that it's not necessary to continue examining dependents after reaching
1721  * an optional_all dependency.  It's not possible for an optional_all dependency
1722  * to change satisfiability without also coming online, in which case we get a
1723  * start event and propagation continues naturally.  However, it does no harm to
1724  * continue propagating satisfiability (as it is a relatively rare event), and
1725  * keeps the walker code simple and generic.
1726  */
1727 /*ARGSUSED*/
1728 static int
1729 satbility_cb(graph_vertex_t *v, void *arg)
1730 {
1731         if (v->gv_type == GVT_INST)
1732                 graph_start_if_satisfied(v);
1733 
1734         return (UU_WALK_NEXT);
1735 }
1736 
1737 static void
1738 propagate_satbility(graph_vertex_t *v)
1739 {
1740         graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1741 }
1742 
1743 static void propagate_stop(graph_vertex_t *, void *);
1744 
1745 /* ARGSUSED */
1746 static void
1747 propagate_start(graph_vertex_t *v, void *arg)
1748 {
1749         switch (v->gv_type) {
1750         case GVT_INST:
1751                 graph_start_if_satisfied(v);
1752                 break;
1753 
1754         case GVT_GROUP:
1755                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1756                         graph_walk_dependents(v, propagate_stop,
1757                             (void *)RERR_RESTART);
1758                         break;
1759                 }
1760                 /* FALLTHROUGH */
1761 
1762         case GVT_SVC:
1763                 graph_walk_dependents(v, propagate_start, NULL);
1764                 break;
1765 
1766         case GVT_FILE:
1767 #ifndef NDEBUG
1768                 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1769                     __FILE__, __LINE__);
1770 #endif
1771                 abort();
1772                 /* NOTREACHED */
1773 
1774         default:
1775 #ifndef NDEBUG
1776                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1777                     v->gv_type);
1778 #endif
1779                 abort();
1780         }
1781 }
1782 
1783 static void
1784 propagate_stop(graph_vertex_t *v, void *arg)
1785 {
1786         graph_edge_t *e;
1787         graph_vertex_t *svc;
1788         restarter_error_t err = (restarter_error_t)arg;
1789 
1790         switch (v->gv_type) {
1791         case GVT_INST:
1792                 /* Restarter */
1793                 if (err > RERR_NONE && inst_running(v)) {
1794                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1795                                 vertex_send_event(v,
1796                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1797                         } else {
1798                                 vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1799                         }
1800                 }
1801                 break;
1802 
1803         case GVT_SVC:
1804                 graph_walk_dependents(v, propagate_stop, arg);
1805                 break;
1806 
1807         case GVT_FILE:
1808 #ifndef NDEBUG
1809                 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1810                     __FILE__, __LINE__);
1811 #endif
1812                 abort();
1813                 /* NOTREACHED */
1814 
1815         case GVT_GROUP:
1816                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1817                         graph_walk_dependents(v, propagate_start, NULL);
1818                         break;
1819                 }
1820 
1821                 if (err == RERR_NONE || err > v->gv_restart)
1822                         break;
1823 
1824                 assert(uu_list_numnodes(v->gv_dependents) == 1);
1825                 e = uu_list_first(v->gv_dependents);
1826                 svc = e->ge_vertex;
1827 
1828                 if (inst_running(svc)) {
1829                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1830                                 vertex_send_event(svc,
1831                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1832                         } else {
1833                                 vertex_send_event(svc,
1834                                     RESTARTER_EVENT_TYPE_STOP);
1835                         }
1836                 }
1837                 break;
1838 
1839         default:
1840 #ifndef NDEBUG
1841                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1842                     v->gv_type);
1843 #endif
1844                 abort();
1845         }
1846 }
1847 
1848 void
1849 offline_vertex(graph_vertex_t *v)
1850 {
1851         scf_handle_t *h = libscf_handle_create_bound_loop();
1852         scf_instance_t *scf_inst = safe_scf_instance_create(h);
1853         scf_propertygroup_t *pg = safe_scf_pg_create(h);
1854         restarter_instance_state_t state, next_state;
1855         int r;
1856 
1857         assert(v->gv_type == GVT_INST);
1858 
1859         if (scf_inst == NULL)
1860                 bad_error("safe_scf_instance_create", scf_error());
1861         if (pg == NULL)
1862                 bad_error("safe_scf_pg_create", scf_error());
1863 
1864         /* if the vertex is already going offline, return */
1865 rep_retry:
1866         if (scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, scf_inst, NULL,
1867             NULL, SCF_DECODE_FMRI_EXACT) != 0) {
1868                 switch (scf_error()) {
1869                 case SCF_ERROR_CONNECTION_BROKEN:
1870                         libscf_handle_rebind(h);
1871                         goto rep_retry;
1872 
1873                 case SCF_ERROR_NOT_FOUND:
1874                         scf_pg_destroy(pg);
1875                         scf_instance_destroy(scf_inst);
1876                         (void) scf_handle_unbind(h);
1877                         scf_handle_destroy(h);
1878                         return;
1879                 }
1880                 uu_die("Can't decode FMRI %s: %s\n", v->gv_name,
1881                     scf_strerror(scf_error()));
1882         }
1883 
1884         r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
1885         if (r != 0) {
1886                 switch (scf_error()) {
1887                 case SCF_ERROR_CONNECTION_BROKEN:
1888                         libscf_handle_rebind(h);
1889                         goto rep_retry;
1890 
1891                 case SCF_ERROR_NOT_SET:
1892                 case SCF_ERROR_NOT_FOUND:
1893                         scf_pg_destroy(pg);
1894                         scf_instance_destroy(scf_inst);
1895                         (void) scf_handle_unbind(h);
1896                         scf_handle_destroy(h);
1897                         return;
1898 
1899                 default:
1900                         bad_error("scf_instance_get_pg", scf_error());
1901                 }
1902         } else {
1903                 r = libscf_read_states(pg, &state, &next_state);
1904                 if (r == 0 && (next_state == RESTARTER_STATE_OFFLINE ||
1905                     next_state == RESTARTER_STATE_DISABLED)) {
1906                         log_framework(LOG_DEBUG,
1907                             "%s: instance is already going down.\n",
1908                             v->gv_name);
1909                         scf_pg_destroy(pg);
1910                         scf_instance_destroy(scf_inst);
1911                         (void) scf_handle_unbind(h);
1912                         scf_handle_destroy(h);
1913                         return;
1914                 }
1915         }
1916 
1917         scf_pg_destroy(pg);
1918         scf_instance_destroy(scf_inst);
1919         (void) scf_handle_unbind(h);
1920         scf_handle_destroy(h);
1921 
1922         vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP_RESET);
1923 }
1924 
1925 /*
1926  * void graph_enable_by_vertex()
1927  *   If admin is non-zero, this is an administrative request for change
1928  *   of the enabled property.  Thus, send the ADMIN_DISABLE rather than
1929  *   a plain DISABLE restarter event.
1930  */
1931 void
1932 graph_enable_by_vertex(graph_vertex_t *vertex, int enable, int admin)
1933 {
1934         graph_vertex_t *v;
1935         int r;
1936 
1937         assert(MUTEX_HELD(&dgraph_lock));
1938         assert((vertex->gv_flags & GV_CONFIGURED));
1939 
1940         vertex->gv_flags = (vertex->gv_flags & ~GV_ENABLED) |
1941             (enable ? GV_ENABLED : 0);
1942 
1943         if (enable) {
1944                 if (vertex->gv_state != RESTARTER_STATE_OFFLINE &&
1945                     vertex->gv_state != RESTARTER_STATE_DEGRADED &&
1946                     vertex->gv_state != RESTARTER_STATE_ONLINE) {
1947                         /*
1948                          * In case the vertex was notified to go down,
1949                          * but now can return online, clear the _TOOFFLINE
1950                          * and _TODISABLE flags.
1951                          */
1952                         vertex->gv_flags &= ~GV_TOOFFLINE;
1953                         vertex->gv_flags &= ~GV_TODISABLE;
1954 
1955                         vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ENABLE);
1956                 }
1957 
1958                 /*
1959                  * Wait for state update from restarter before sending _START or
1960                  * _STOP.
1961                  */
1962 
1963                 return;
1964         }
1965 
1966         if (vertex->gv_state == RESTARTER_STATE_DISABLED)
1967                 return;
1968 
1969         if (!admin) {
1970                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_DISABLE);
1971 
1972                 /*
1973                  * Wait for state update from restarter before sending _START or
1974                  * _STOP.
1975                  */
1976 
1977                 return;
1978         }
1979 
1980         /*
1981          * If it is a DISABLE event requested by the administrator then we are
1982          * offlining the dependents first.
1983          */
1984 
1985         /*
1986          * Set GV_TOOFFLINE for the services we are offlining. We cannot
1987          * clear the GV_TOOFFLINE bits from all the services because
1988          * other DISABLE events might be handled at the same time.
1989          */
1990         vertex->gv_flags |= GV_TOOFFLINE;
1991 
1992         /* remember which vertex to disable... */
1993         vertex->gv_flags |= GV_TODISABLE;
1994 
1995         log_framework(LOG_DEBUG, "Marking in-subtree vertices before "
1996             "disabling %s.\n", vertex->gv_name);
1997 
1998         /* set GV_TOOFFLINE for its dependents */
1999         r = uu_list_walk(vertex->gv_dependents, (uu_walk_fn_t *)mark_subtree,
2000             NULL, 0);
2001         assert(r == 0);
2002 
2003         /* disable the instance now if there is nothing else to offline */
2004         if (insubtree_dependents_down(vertex) == B_TRUE) {
2005                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
2006                 return;
2007         }
2008 
2009         /*
2010          * This loop is similar to the one used for the graph reversal shutdown
2011          * and could be improved in term of performance for the subtree reversal
2012          * disable case.
2013          */
2014         for (v = uu_list_first(dgraph); v != NULL;
2015             v = uu_list_next(dgraph, v)) {
2016                 /* skip the vertex we are disabling for now */
2017                 if (v == vertex)
2018                         continue;
2019 
2020                 if (v->gv_type != GVT_INST ||
2021                     (v->gv_flags & GV_CONFIGURED) == 0 ||
2022                     (v->gv_flags & GV_ENABLED) == 0 ||
2023                     (v->gv_flags & GV_TOOFFLINE) == 0)
2024                         continue;
2025 
2026                 if ((v->gv_state != RESTARTER_STATE_ONLINE) &&
2027                     (v->gv_state != RESTARTER_STATE_DEGRADED)) {
2028                         /* continue if there is nothing to offline */
2029                         continue;
2030                 }
2031 
2032                 /*
2033                  * Instances which are up need to come down before we're
2034                  * done, but we can only offline the leaves here. An
2035                  * instance is a leaf when all its dependents are down.
2036                  */
2037                 if (insubtree_dependents_down(v) == B_TRUE) {
2038                         log_framework(LOG_DEBUG, "Offlining in-subtree "
2039                             "instance %s for %s.\n",
2040                             v->gv_name, vertex->gv_name);
2041                         offline_vertex(v);
2042                 }
2043         }
2044 }
2045 
2046 static int configure_vertex(graph_vertex_t *, scf_instance_t *);
2047 
2048 /*
2049  * Set the restarter for v to fmri_arg.  That is, make sure a vertex for
2050  * fmri_arg exists, make v depend on it, and send _ADD_INSTANCE for v.  If
2051  * v is already configured and fmri_arg indicates the current restarter, do
2052  * nothing.  If v is configured and fmri_arg is a new restarter, delete v's
2053  * dependency on the restarter, send _REMOVE_INSTANCE for v, and set the new
2054  * restarter.  Returns 0 on success, EINVAL if the FMRI is invalid,
2055  * ECONNABORTED if the repository connection is broken, and ELOOP
2056  * if the dependency would create a cycle.  In the last case, *pathp will
2057  * point to a -1-terminated array of ids which compose the path from v to
2058  * restarter_fmri.
2059  */
2060 int
2061 graph_change_restarter(graph_vertex_t *v, const char *fmri_arg, scf_handle_t *h,
2062     int **pathp)
2063 {
2064         char *restarter_fmri = NULL;
2065         graph_vertex_t *rv;
2066         int err;
2067         int id;
2068 
2069         assert(MUTEX_HELD(&dgraph_lock));
2070 
2071         if (fmri_arg[0] != '\0') {
2072                 err = fmri_canonify(fmri_arg, &restarter_fmri, B_TRUE);
2073                 if (err != 0) {
2074                         assert(err == EINVAL);
2075                         return (err);
2076                 }
2077         }
2078 
2079         if (restarter_fmri == NULL ||
2080             strcmp(restarter_fmri, SCF_SERVICE_STARTD) == 0) {
2081                 if (v->gv_flags & GV_CONFIGURED) {
2082                         if (v->gv_restarter_id == -1) {
2083                                 if (restarter_fmri != NULL)
2084                                         startd_free(restarter_fmri,
2085                                             max_scf_fmri_size);
2086                                 return (0);
2087                         }
2088 
2089                         graph_unset_restarter(v);
2090                 }
2091 
2092                 /* Master restarter, nothing to do. */
2093                 v->gv_restarter_id = -1;
2094                 v->gv_restarter_channel = NULL;
2095                 vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2096                 return (0);
2097         }
2098 
2099         if (v->gv_flags & GV_CONFIGURED) {
2100                 id = dict_lookup_byname(restarter_fmri);
2101                 if (id != -1 && v->gv_restarter_id == id) {
2102                         startd_free(restarter_fmri, max_scf_fmri_size);
2103                         return (0);
2104                 }
2105 
2106                 graph_unset_restarter(v);
2107         }
2108 
2109         err = graph_insert_vertex_unconfigured(restarter_fmri, GVT_INST, 0,
2110             RERR_NONE, &rv);
2111         startd_free(restarter_fmri, max_scf_fmri_size);
2112         assert(err == 0 || err == EEXIST);
2113 
2114         if (rv->gv_delegate_initialized == 0) {
2115                 if ((rv->gv_delegate_channel = restarter_protocol_init_delegate(
2116                     rv->gv_name)) == NULL)
2117                         return (EINVAL);
2118                 rv->gv_delegate_initialized = 1;
2119         }
2120         v->gv_restarter_id = rv->gv_id;
2121         v->gv_restarter_channel = rv->gv_delegate_channel;
2122 
2123         err = graph_insert_dependency(v, rv, pathp);
2124         if (err != 0) {
2125                 assert(err == ELOOP);
2126                 return (ELOOP);
2127         }
2128 
2129         vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2130 
2131         if (!(rv->gv_flags & GV_CONFIGURED)) {
2132                 scf_instance_t *inst;
2133 
2134                 err = libscf_fmri_get_instance(h, rv->gv_name, &inst);
2135                 switch (err) {
2136                 case 0:
2137                         err = configure_vertex(rv, inst);
2138                         scf_instance_destroy(inst);
2139                         switch (err) {
2140                         case 0:
2141                         case ECANCELED:
2142                                 break;
2143 
2144                         case ECONNABORTED:
2145                                 return (ECONNABORTED);
2146 
2147                         default:
2148                                 bad_error("configure_vertex", err);
2149                         }
2150                         break;
2151 
2152                 case ECONNABORTED:
2153                         return (ECONNABORTED);
2154 
2155                 case ENOENT:
2156                         break;
2157 
2158                 case ENOTSUP:
2159                         /*
2160                          * The fmri doesn't specify an instance - translate
2161                          * to EINVAL.
2162                          */
2163                         return (EINVAL);
2164 
2165                 case EINVAL:
2166                 default:
2167                         bad_error("libscf_fmri_get_instance", err);
2168                 }
2169         }
2170 
2171         return (0);
2172 }
2173 
2174 
2175 /*
2176  * Add all of the instances of the service named by fmri to the graph.
2177  * Returns
2178  *   0 - success
2179  *   ENOENT - service indicated by fmri does not exist
2180  *
2181  * In both cases *reboundp will be B_TRUE if the handle was rebound, or B_FALSE
2182  * otherwise.
2183  */
2184 static int
2185 add_service(const char *fmri, scf_handle_t *h, boolean_t *reboundp)
2186 {
2187         scf_service_t *svc;
2188         scf_instance_t *inst;
2189         scf_iter_t *iter;
2190         char *inst_fmri;
2191         int ret, r;
2192 
2193         *reboundp = B_FALSE;
2194 
2195         svc = safe_scf_service_create(h);
2196         inst = safe_scf_instance_create(h);
2197         iter = safe_scf_iter_create(h);
2198         inst_fmri = startd_alloc(max_scf_fmri_size);
2199 
2200 rebound:
2201         if (scf_handle_decode_fmri(h, fmri, NULL, svc, NULL, NULL, NULL,
2202             SCF_DECODE_FMRI_EXACT) != 0) {
2203                 switch (scf_error()) {
2204                 case SCF_ERROR_CONNECTION_BROKEN:
2205                 default:
2206                         libscf_handle_rebind(h);
2207                         *reboundp = B_TRUE;
2208                         goto rebound;
2209 
2210                 case SCF_ERROR_NOT_FOUND:
2211                         ret = ENOENT;
2212                         goto out;
2213 
2214                 case SCF_ERROR_INVALID_ARGUMENT:
2215                 case SCF_ERROR_CONSTRAINT_VIOLATED:
2216                 case SCF_ERROR_NOT_BOUND:
2217                 case SCF_ERROR_HANDLE_MISMATCH:
2218                         bad_error("scf_handle_decode_fmri", scf_error());
2219                 }
2220         }
2221 
2222         if (scf_iter_service_instances(iter, svc) != 0) {
2223                 switch (scf_error()) {
2224                 case SCF_ERROR_CONNECTION_BROKEN:
2225                 default:
2226                         libscf_handle_rebind(h);
2227                         *reboundp = B_TRUE;
2228                         goto rebound;
2229 
2230                 case SCF_ERROR_DELETED:
2231                         ret = ENOENT;
2232                         goto out;
2233 
2234                 case SCF_ERROR_HANDLE_MISMATCH:
2235                 case SCF_ERROR_NOT_BOUND:
2236                 case SCF_ERROR_NOT_SET:
2237                         bad_error("scf_iter_service_instances", scf_error());
2238                 }
2239         }
2240 
2241         for (;;) {
2242                 r = scf_iter_next_instance(iter, inst);
2243                 if (r == 0)
2244                         break;
2245                 if (r != 1) {
2246                         switch (scf_error()) {
2247                         case SCF_ERROR_CONNECTION_BROKEN:
2248                         default:
2249                                 libscf_handle_rebind(h);
2250                                 *reboundp = B_TRUE;
2251                                 goto rebound;
2252 
2253                         case SCF_ERROR_DELETED:
2254                                 ret = ENOENT;
2255                                 goto out;
2256 
2257                         case SCF_ERROR_HANDLE_MISMATCH:
2258                         case SCF_ERROR_NOT_BOUND:
2259                         case SCF_ERROR_NOT_SET:
2260                         case SCF_ERROR_INVALID_ARGUMENT:
2261                                 bad_error("scf_iter_next_instance",
2262                                     scf_error());
2263                         }
2264                 }
2265 
2266                 if (scf_instance_to_fmri(inst, inst_fmri, max_scf_fmri_size) <
2267                     0) {
2268                         switch (scf_error()) {
2269                         case SCF_ERROR_CONNECTION_BROKEN:
2270                                 libscf_handle_rebind(h);
2271                                 *reboundp = B_TRUE;
2272                                 goto rebound;
2273 
2274                         case SCF_ERROR_DELETED:
2275                                 continue;
2276 
2277                         case SCF_ERROR_NOT_BOUND:
2278                         case SCF_ERROR_NOT_SET:
2279                                 bad_error("scf_instance_to_fmri", scf_error());
2280                         }
2281                 }
2282 
2283                 r = dgraph_add_instance(inst_fmri, inst, B_FALSE);
2284                 switch (r) {
2285                 case 0:
2286                 case ECANCELED:
2287                         break;
2288 
2289                 case EEXIST:
2290                         continue;
2291 
2292                 case ECONNABORTED:
2293                         libscf_handle_rebind(h);
2294                         *reboundp = B_TRUE;
2295                         goto rebound;
2296 
2297                 case EINVAL:
2298                 default:
2299                         bad_error("dgraph_add_instance", r);
2300                 }
2301         }
2302 
2303         ret = 0;
2304 
2305 out:
2306         startd_free(inst_fmri, max_scf_fmri_size);
2307         scf_iter_destroy(iter);
2308         scf_instance_destroy(inst);
2309         scf_service_destroy(svc);
2310         return (ret);
2311 }
2312 
2313 struct depfmri_info {
2314         graph_vertex_t  *v;             /* GVT_GROUP vertex */
2315         gv_type_t       type;           /* type of dependency */
2316         const char      *inst_fmri;     /* FMRI of parental GVT_INST vert. */
2317         const char      *pg_name;       /* Name of dependency pg */
2318         scf_handle_t    *h;
2319         int             err;            /* return error code */
2320         int             **pathp;        /* return circular dependency path */
2321 };
2322 
2323 /*
2324  * Find or create a vertex for fmri and make info->v depend on it.
2325  * Returns
2326  *   0 - success
2327  *   nonzero - failure
2328  *
2329  * On failure, sets info->err to
2330  *   EINVAL - fmri is invalid
2331  *            fmri does not match info->type
2332  *   ELOOP - Adding the dependency creates a circular dependency.  *info->pathp
2333  *           will point to an array of the ids of the members of the cycle.
2334  *   ECONNABORTED - repository connection was broken
2335  *   ECONNRESET - succeeded, but repository connection was reset
2336  */
2337 static int
2338 process_dependency_fmri(const char *fmri, struct depfmri_info *info)
2339 {
2340         int err;
2341         graph_vertex_t *depgroup_v, *v;
2342         char *fmri_copy, *cfmri;
2343         size_t fmri_copy_sz;
2344         const char *scope, *service, *instance, *pg;
2345         scf_instance_t *inst;
2346         boolean_t rebound;
2347 
2348         assert(MUTEX_HELD(&dgraph_lock));
2349 
2350         /* Get or create vertex for FMRI */
2351         depgroup_v = info->v;
2352 
2353         if (strncmp(fmri, "file:", sizeof ("file:") - 1) == 0) {
2354                 if (info->type != GVT_FILE) {
2355                         log_framework(LOG_NOTICE,
2356                             "FMRI \"%s\" is not allowed for the \"%s\" "
2357                             "dependency's type of instance %s.\n", fmri,
2358                             info->pg_name, info->inst_fmri);
2359                         return (info->err = EINVAL);
2360                 }
2361 
2362                 err = graph_insert_vertex_unconfigured(fmri, info->type, 0,
2363                     RERR_NONE, &v);
2364                 switch (err) {
2365                 case 0:
2366                         break;
2367 
2368                 case EEXIST:
2369                         assert(v->gv_type == GVT_FILE);
2370                         break;
2371 
2372                 case EINVAL:            /* prevented above */
2373                 default:
2374                         bad_error("graph_insert_vertex_unconfigured", err);
2375                 }
2376         } else {
2377                 if (info->type != GVT_INST) {
2378                         log_framework(LOG_NOTICE,
2379                             "FMRI \"%s\" is not allowed for the \"%s\" "
2380                             "dependency's type of instance %s.\n", fmri,
2381                             info->pg_name, info->inst_fmri);
2382                         return (info->err = EINVAL);
2383                 }
2384 
2385                 /*
2386                  * We must canonify fmri & add a vertex for it.
2387                  */
2388                 fmri_copy_sz = strlen(fmri) + 1;
2389                 fmri_copy = startd_alloc(fmri_copy_sz);
2390                 (void) strcpy(fmri_copy, fmri);
2391 
2392                 /* Determine if the FMRI is a property group or instance */
2393                 if (scf_parse_svc_fmri(fmri_copy, &scope, &service,
2394                     &instance, &pg, NULL) != 0) {
2395                         startd_free(fmri_copy, fmri_copy_sz);
2396                         log_framework(LOG_NOTICE,
2397                             "Dependency \"%s\" of %s has invalid FMRI "
2398                             "\"%s\".\n", info->pg_name, info->inst_fmri,
2399                             fmri);
2400                         return (info->err = EINVAL);
2401                 }
2402 
2403                 if (service == NULL || pg != NULL) {
2404                         startd_free(fmri_copy, fmri_copy_sz);
2405                         log_framework(LOG_NOTICE,
2406                             "Dependency \"%s\" of %s does not designate a "
2407                             "service or instance.\n", info->pg_name,
2408                             info->inst_fmri);
2409                         return (info->err = EINVAL);
2410                 }
2411 
2412                 if (scope == NULL || strcmp(scope, SCF_SCOPE_LOCAL) == 0) {
2413                         cfmri = uu_msprintf("svc:/%s%s%s",
2414                             service, instance ? ":" : "", instance ? instance :
2415                             "");
2416                 } else {
2417                         cfmri = uu_msprintf("svc://%s/%s%s%s",
2418                             scope, service, instance ? ":" : "", instance ?
2419                             instance : "");
2420                 }
2421 
2422                 startd_free(fmri_copy, fmri_copy_sz);
2423 
2424                 err = graph_insert_vertex_unconfigured(cfmri, instance ?
2425                     GVT_INST : GVT_SVC, instance ? 0 : DEPGRP_REQUIRE_ANY,
2426                     RERR_NONE, &v);
2427                 uu_free(cfmri);
2428                 switch (err) {
2429                 case 0:
2430                         break;
2431 
2432                 case EEXIST:
2433                         /* Verify v. */
2434                         if (instance != NULL)
2435                                 assert(v->gv_type == GVT_INST);
2436                         else
2437                                 assert(v->gv_type == GVT_SVC);
2438                         break;
2439 
2440                 default:
2441                         bad_error("graph_insert_vertex_unconfigured", err);
2442                 }
2443         }
2444 
2445         /* Add dependency from depgroup_v to new vertex */
2446         info->err = graph_insert_dependency(depgroup_v, v, info->pathp);
2447         switch (info->err) {
2448         case 0:
2449                 break;
2450 
2451         case ELOOP:
2452                 return (ELOOP);
2453 
2454         default:
2455                 bad_error("graph_insert_dependency", info->err);
2456         }
2457 
2458         /* This must be after we insert the dependency, to avoid looping. */
2459         switch (v->gv_type) {
2460         case GVT_INST:
2461                 if ((v->gv_flags & GV_CONFIGURED) != 0)
2462                         break;
2463 
2464                 inst = safe_scf_instance_create(info->h);
2465 
2466                 rebound = B_FALSE;
2467 
2468 rebound:
2469                 err = libscf_lookup_instance(v->gv_name, inst);
2470                 switch (err) {
2471                 case 0:
2472                         err = configure_vertex(v, inst);
2473                         switch (err) {
2474                         case 0:
2475                         case ECANCELED:
2476                                 break;
2477 
2478                         case ECONNABORTED:
2479                                 libscf_handle_rebind(info->h);
2480                                 rebound = B_TRUE;
2481                                 goto rebound;
2482 
2483                         default:
2484                                 bad_error("configure_vertex", err);
2485                         }
2486                         break;
2487 
2488                 case ENOENT:
2489                         break;
2490 
2491                 case ECONNABORTED:
2492                         libscf_handle_rebind(info->h);
2493                         rebound = B_TRUE;
2494                         goto rebound;
2495 
2496                 case EINVAL:
2497                 case ENOTSUP:
2498                 default:
2499                         bad_error("libscf_fmri_get_instance", err);
2500                 }
2501 
2502                 scf_instance_destroy(inst);
2503 
2504                 if (rebound)
2505                         return (info->err = ECONNRESET);
2506                 break;
2507 
2508         case GVT_SVC:
2509                 (void) add_service(v->gv_name, info->h, &rebound);
2510                 if (rebound)
2511                         return (info->err = ECONNRESET);
2512         }
2513 
2514         return (0);
2515 }
2516 
2517 struct deppg_info {
2518         graph_vertex_t  *v;             /* GVT_INST vertex */
2519         int             err;            /* return error */
2520         int             **pathp;        /* return circular dependency path */
2521 };
2522 
2523 /*
2524  * Make info->v depend on a new GVT_GROUP node for this property group,
2525  * and then call process_dependency_fmri() for the values of the entity
2526  * property.  Return 0 on success, or if something goes wrong return nonzero
2527  * and set info->err to ECONNABORTED, EINVAL, or the error code returned by
2528  * process_dependency_fmri().
2529  */
2530 static int
2531 process_dependency_pg(scf_propertygroup_t *pg, struct deppg_info *info)
2532 {
2533         scf_handle_t *h;
2534         depgroup_type_t deptype;
2535         restarter_error_t rerr;
2536         struct depfmri_info linfo;
2537         char *fmri, *pg_name;
2538         size_t fmri_sz;
2539         graph_vertex_t *depgrp;
2540         scf_property_t *prop;
2541         int err;
2542         int empty;
2543         scf_error_t scferr;
2544         ssize_t len;
2545 
2546         assert(MUTEX_HELD(&dgraph_lock));
2547 
2548         h = scf_pg_handle(pg);
2549 
2550         pg_name = startd_alloc(max_scf_name_size);
2551 
2552         len = scf_pg_get_name(pg, pg_name, max_scf_name_size);
2553         if (len < 0) {
2554                 startd_free(pg_name, max_scf_name_size);
2555                 switch (scf_error()) {
2556                 case SCF_ERROR_CONNECTION_BROKEN:
2557                 default:
2558                         return (info->err = ECONNABORTED);
2559 
2560                 case SCF_ERROR_DELETED:
2561                         return (info->err = 0);
2562 
2563                 case SCF_ERROR_NOT_SET:
2564                         bad_error("scf_pg_get_name", scf_error());
2565                 }
2566         }
2567 
2568         /*
2569          * Skip over empty dependency groups.  Since dependency property
2570          * groups are updated atomically, they are either empty or
2571          * fully populated.
2572          */
2573         empty = depgroup_empty(h, pg);
2574         if (empty < 0) {
2575                 log_error(LOG_INFO,
2576                     "Error reading dependency group \"%s\" of %s: %s\n",
2577                     pg_name, info->v->gv_name, scf_strerror(scf_error()));
2578                 startd_free(pg_name, max_scf_name_size);
2579                 return (info->err = EINVAL);
2580 
2581         } else if (empty == 1) {
2582                 log_framework(LOG_DEBUG,
2583                     "Ignoring empty dependency group \"%s\" of %s\n",
2584                     pg_name, info->v->gv_name);
2585                 startd_free(pg_name, max_scf_name_size);
2586                 return (info->err = 0);
2587         }
2588 
2589         fmri_sz = strlen(info->v->gv_name) + 1 + len + 1;
2590         fmri = startd_alloc(fmri_sz);
2591 
2592         (void) snprintf(fmri, fmri_sz, "%s>%s", info->v->gv_name,
2593             pg_name);
2594 
2595         /* Validate the pg before modifying the graph */
2596         deptype = depgroup_read_grouping(h, pg);
2597         if (deptype == DEPGRP_UNSUPPORTED) {
2598                 log_error(LOG_INFO,
2599                     "Dependency \"%s\" of %s has an unknown grouping value.\n",
2600                     pg_name, info->v->gv_name);
2601                 startd_free(fmri, fmri_sz);
2602                 startd_free(pg_name, max_scf_name_size);
2603                 return (info->err = EINVAL);
2604         }
2605 
2606         rerr = depgroup_read_restart(h, pg);
2607         if (rerr == RERR_UNSUPPORTED) {
2608                 log_error(LOG_INFO,
2609                     "Dependency \"%s\" of %s has an unknown restart_on value."
2610                     "\n", pg_name, info->v->gv_name);
2611                 startd_free(fmri, fmri_sz);
2612                 startd_free(pg_name, max_scf_name_size);
2613                 return (info->err = EINVAL);
2614         }
2615 
2616         prop = safe_scf_property_create(h);
2617 
2618         if (scf_pg_get_property(pg, SCF_PROPERTY_ENTITIES, prop) != 0) {
2619                 scferr = scf_error();
2620                 scf_property_destroy(prop);
2621                 if (scferr == SCF_ERROR_DELETED) {
2622                         startd_free(fmri, fmri_sz);
2623                         startd_free(pg_name, max_scf_name_size);
2624                         return (info->err = 0);
2625                 } else if (scferr != SCF_ERROR_NOT_FOUND) {
2626                         startd_free(fmri, fmri_sz);
2627                         startd_free(pg_name, max_scf_name_size);
2628                         return (info->err = ECONNABORTED);
2629                 }
2630 
2631                 log_error(LOG_INFO,
2632                     "Dependency \"%s\" of %s is missing a \"%s\" property.\n",
2633                     pg_name, info->v->gv_name, SCF_PROPERTY_ENTITIES);
2634 
2635                 startd_free(fmri, fmri_sz);
2636                 startd_free(pg_name, max_scf_name_size);
2637 
2638                 return (info->err = EINVAL);
2639         }
2640 
2641         /* Create depgroup vertex for pg */
2642         err = graph_insert_vertex_unconfigured(fmri, GVT_GROUP, deptype,
2643             rerr, &depgrp);
2644         assert(err == 0);
2645         startd_free(fmri, fmri_sz);
2646 
2647         /* Add dependency from inst vertex to new vertex */
2648         err = graph_insert_dependency(info->v, depgrp, info->pathp);
2649         /* ELOOP can't happen because this should be a new vertex */
2650         assert(err == 0);
2651 
2652         linfo.v = depgrp;
2653         linfo.type = depgroup_read_scheme(h, pg);
2654         linfo.inst_fmri = info->v->gv_name;
2655         linfo.pg_name = pg_name;
2656         linfo.h = h;
2657         linfo.err = 0;
2658         linfo.pathp = info->pathp;
2659         err = walk_property_astrings(prop, (callback_t)process_dependency_fmri,
2660             &linfo);
2661 
2662         scf_property_destroy(prop);
2663         startd_free(pg_name, max_scf_name_size);
2664 
2665         switch (err) {
2666         case 0:
2667         case EINTR:
2668                 return (info->err = linfo.err);
2669 
2670         case ECONNABORTED:
2671         case EINVAL:
2672                 return (info->err = err);
2673 
2674         case ECANCELED:
2675                 return (info->err = 0);
2676 
2677         case ECONNRESET:
2678                 return (info->err = ECONNABORTED);
2679 
2680         default:
2681                 bad_error("walk_property_astrings", err);
2682                 /* NOTREACHED */
2683         }
2684 }
2685 
2686 /*
2687  * Build the dependency info for v from the repository.  Returns 0 on success,
2688  * ECONNABORTED on repository disconnection, EINVAL if the repository
2689  * configuration is invalid, and ELOOP if a dependency would cause a cycle.
2690  * In the last case, *pathp will point to a -1-terminated array of ids which
2691  * constitute the rest of the dependency cycle.
2692  */
2693 static int
2694 set_dependencies(graph_vertex_t *v, scf_instance_t *inst, int **pathp)
2695 {
2696         struct deppg_info info;
2697         int err;
2698         uint_t old_configured;
2699 
2700         assert(MUTEX_HELD(&dgraph_lock));
2701 
2702         /*
2703          * Mark the vertex as configured during dependency insertion to avoid
2704          * dependency cycles (which can appear in the graph if one of the
2705          * vertices is an exclusion-group).
2706          */
2707         old_configured = v->gv_flags & GV_CONFIGURED;
2708         v->gv_flags |= GV_CONFIGURED;
2709 
2710         info.err = 0;
2711         info.v = v;
2712         info.pathp = pathp;
2713 
2714         err = walk_dependency_pgs(inst, (callback_t)process_dependency_pg,
2715             &info);
2716 
2717         if (!old_configured)
2718                 v->gv_flags &= ~GV_CONFIGURED;
2719 
2720         switch (err) {
2721         case 0:
2722         case EINTR:
2723                 return (info.err);
2724 
2725         case ECONNABORTED:
2726                 return (ECONNABORTED);
2727 
2728         case ECANCELED:
2729                 /* Should get delete event, so return 0. */
2730                 return (0);
2731 
2732         default:
2733                 bad_error("walk_dependency_pgs", err);
2734                 /* NOTREACHED */
2735         }
2736 }
2737 
2738 
2739 static void
2740 handle_cycle(const char *fmri, int *path)
2741 {
2742         const char *cp;
2743         size_t sz;
2744 
2745         assert(MUTEX_HELD(&dgraph_lock));
2746 
2747         path_to_str(path, (char **)&cp, &sz);
2748 
2749         log_error(LOG_ERR, "Transitioning %s to maintenance "
2750             "because it completes a dependency cycle (see svcs -xv for "
2751             "details):\n%s", fmri ? fmri : "?", cp);
2752 
2753         startd_free((void *)cp, sz);
2754 }
2755 
2756 /*
2757  * Increment the vertex's reference count to prevent the vertex removal
2758  * from the dgraph.
2759  */
2760 static void
2761 vertex_ref(graph_vertex_t *v)
2762 {
2763         assert(MUTEX_HELD(&dgraph_lock));
2764 
2765         v->gv_refs++;
2766 }
2767 
2768 /*
2769  * Decrement the vertex's reference count and remove the vertex from
2770  * the dgraph when possible.
2771  *
2772  * Return VERTEX_REMOVED when the vertex has been removed otherwise
2773  * return VERTEX_INUSE.
2774  */
2775 static int
2776 vertex_unref(graph_vertex_t *v)
2777 {
2778         assert(MUTEX_HELD(&dgraph_lock));
2779         assert(v->gv_refs > 0);
2780 
2781         v->gv_refs--;
2782 
2783         return (free_if_unrefed(v));
2784 }
2785 
2786 /*
2787  * When run on the dependencies of a vertex, populates list with
2788  * graph_edge_t's which point to the service vertices or the instance
2789  * vertices (no GVT_GROUP nodes) on which the vertex depends.
2790  *
2791  * Increment the vertex's reference count once the vertex is inserted
2792  * in the list. The vertex won't be able to be deleted from the dgraph
2793  * while it is referenced.
2794  */
2795 static int
2796 append_svcs_or_insts(graph_edge_t *e, uu_list_t *list)
2797 {
2798         graph_vertex_t *v = e->ge_vertex;
2799         graph_edge_t *new;
2800         int r;
2801 
2802         switch (v->gv_type) {
2803         case GVT_INST:
2804         case GVT_SVC:
2805                 break;
2806 
2807         case GVT_GROUP:
2808                 r = uu_list_walk(v->gv_dependencies,
2809                     (uu_walk_fn_t *)append_svcs_or_insts, list, 0);
2810                 assert(r == 0);
2811                 return (UU_WALK_NEXT);
2812 
2813         case GVT_FILE:
2814                 return (UU_WALK_NEXT);
2815 
2816         default:
2817 #ifndef NDEBUG
2818                 uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
2819                     __LINE__, v->gv_type);
2820 #endif
2821                 abort();
2822         }
2823 
2824         new = startd_alloc(sizeof (*new));
2825         new->ge_vertex = v;
2826         uu_list_node_init(new, &new->ge_link, graph_edge_pool);
2827         r = uu_list_insert_before(list, NULL, new);
2828         assert(r == 0);
2829 
2830         /*
2831          * Because we are inserting the vertex in a list, we don't want
2832          * the vertex to be freed while the list is in use. In order to
2833          * achieve that, increment the vertex's reference count.
2834          */
2835         vertex_ref(v);
2836 
2837         return (UU_WALK_NEXT);
2838 }
2839 
2840 static boolean_t
2841 should_be_in_subgraph(graph_vertex_t *v)
2842 {
2843         graph_edge_t *e;
2844 
2845         if (v == milestone)
2846                 return (B_TRUE);
2847 
2848         /*
2849          * v is in the subgraph if any of its dependents are in the subgraph.
2850          * Except for EXCLUDE_ALL dependents.  And OPTIONAL dependents only
2851          * count if we're enabled.
2852          */
2853         for (e = uu_list_first(v->gv_dependents);
2854             e != NULL;
2855             e = uu_list_next(v->gv_dependents, e)) {
2856                 graph_vertex_t *dv = e->ge_vertex;
2857 
2858                 if (!(dv->gv_flags & GV_INSUBGRAPH))
2859                         continue;
2860 
2861                 /*
2862                  * Don't include instances that are optional and disabled.
2863                  */
2864                 if (v->gv_type == GVT_INST && dv->gv_type == GVT_SVC) {
2865 
2866                         int in = 0;
2867                         graph_edge_t *ee;
2868 
2869                         for (ee = uu_list_first(dv->gv_dependents);
2870                             ee != NULL;
2871                             ee = uu_list_next(dv->gv_dependents, ee)) {
2872 
2873                                 graph_vertex_t *ddv = e->ge_vertex;
2874 
2875                                 if (ddv->gv_type == GVT_GROUP &&
2876                                     ddv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2877                                         continue;
2878 
2879                                 if (ddv->gv_type == GVT_GROUP &&
2880                                     ddv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2881                                     !(v->gv_flags & GV_ENBLD_NOOVR))
2882                                         continue;
2883 
2884                                 in = 1;
2885                         }
2886                         if (!in)
2887                                 continue;
2888                 }
2889                 if (v->gv_type == GVT_INST &&
2890                     dv->gv_type == GVT_GROUP &&
2891                     dv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2892                     !(v->gv_flags & GV_ENBLD_NOOVR))
2893                         continue;
2894 
2895                 /* Don't include excluded services and instances */
2896                 if (dv->gv_type == GVT_GROUP &&
2897                     dv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2898                         continue;
2899 
2900                 return (B_TRUE);
2901         }
2902 
2903         return (B_FALSE);
2904 }
2905 
2906 /*
2907  * Ensures that GV_INSUBGRAPH is set properly for v and its descendents.  If
2908  * any bits change, manipulate the repository appropriately.  Returns 0 or
2909  * ECONNABORTED.
2910  */
2911 static int
2912 eval_subgraph(graph_vertex_t *v, scf_handle_t *h)
2913 {
2914         boolean_t old = (v->gv_flags & GV_INSUBGRAPH) != 0;
2915         boolean_t new;
2916         graph_edge_t *e;
2917         scf_instance_t *inst;
2918         int ret = 0, r;
2919 
2920         assert(milestone != NULL && milestone != MILESTONE_NONE);
2921 
2922         new = should_be_in_subgraph(v);
2923 
2924         if (new == old)
2925                 return (0);
2926 
2927         log_framework(LOG_DEBUG, new ? "Adding %s to the subgraph.\n" :
2928             "Removing %s from the subgraph.\n", v->gv_name);
2929 
2930         v->gv_flags = (v->gv_flags & ~GV_INSUBGRAPH) |
2931             (new ? GV_INSUBGRAPH : 0);
2932 
2933         if (v->gv_type == GVT_INST && (v->gv_flags & GV_CONFIGURED)) {
2934                 int err;
2935 
2936 get_inst:
2937                 err = libscf_fmri_get_instance(h, v->gv_name, &inst);
2938                 if (err != 0) {
2939                         switch (err) {
2940                         case ECONNABORTED:
2941                                 libscf_handle_rebind(h);
2942                                 ret = ECONNABORTED;
2943                                 goto get_inst;
2944 
2945                         case ENOENT:
2946                                 break;
2947 
2948                         case EINVAL:
2949                         case ENOTSUP:
2950                         default:
2951                                 bad_error("libscf_fmri_get_instance", err);
2952                         }
2953                 } else {
2954                         const char *f;
2955 
2956                         if (new) {
2957                                 err = libscf_delete_enable_ovr(inst);
2958                                 f = "libscf_delete_enable_ovr";
2959                         } else {
2960                                 err = libscf_set_enable_ovr(inst, 0);
2961                                 f = "libscf_set_enable_ovr";
2962                         }
2963                         scf_instance_destroy(inst);
2964                         switch (err) {
2965                         case 0:
2966                         case ECANCELED:
2967                                 break;
2968 
2969                         case ECONNABORTED:
2970                                 libscf_handle_rebind(h);
2971                                 /*
2972                                  * We must continue so the graph is updated,
2973                                  * but we must return ECONNABORTED so any
2974                                  * libscf state held by any callers is reset.
2975                                  */
2976                                 ret = ECONNABORTED;
2977                                 goto get_inst;
2978 
2979                         case EROFS:
2980                         case EPERM:
2981                                 log_error(LOG_WARNING,
2982                                     "Could not set %s/%s for %s: %s.\n",
2983                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
2984                                     v->gv_name, strerror(err));
2985                                 break;
2986 
2987                         default:
2988                                 bad_error(f, err);
2989                         }
2990                 }
2991         }
2992 
2993         for (e = uu_list_first(v->gv_dependencies);
2994             e != NULL;
2995             e = uu_list_next(v->gv_dependencies, e)) {
2996                 r = eval_subgraph(e->ge_vertex, h);
2997                 if (r != 0) {
2998                         assert(r == ECONNABORTED);
2999                         ret = ECONNABORTED;
3000                 }
3001         }
3002 
3003         return (ret);
3004 }
3005 
3006 /*
3007  * Delete the (property group) dependencies of v & create new ones based on
3008  * inst.  If doing so would create a cycle, log a message and put the instance
3009  * into maintenance.  Update GV_INSUBGRAPH flags as necessary.  Returns 0 or
3010  * ECONNABORTED.
3011  */
3012 int
3013 refresh_vertex(graph_vertex_t *v, scf_instance_t *inst)
3014 {
3015         int err;
3016         int *path;
3017         char *fmri;
3018         int r;
3019         scf_handle_t *h = scf_instance_handle(inst);
3020         uu_list_t *old_deps;
3021         int ret = 0;
3022         graph_edge_t *e;
3023         graph_vertex_t *vv;
3024 
3025         assert(MUTEX_HELD(&dgraph_lock));
3026         assert(v->gv_type == GVT_INST);
3027 
3028         log_framework(LOG_DEBUG, "Graph engine: Refreshing %s.\n", v->gv_name);
3029 
3030         if (milestone > MILESTONE_NONE) {
3031                 /*
3032                  * In case some of v's dependencies are being deleted we must
3033                  * make a list of them now for GV_INSUBGRAPH-flag evaluation
3034                  * after the new dependencies are in place.
3035                  */
3036                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
3037 
3038                 err = uu_list_walk(v->gv_dependencies,
3039                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
3040                 assert(err == 0);
3041         }
3042 
3043         delete_instance_dependencies(v, B_FALSE);
3044 
3045         err = set_dependencies(v, inst, &path);
3046         switch (err) {
3047         case 0:
3048                 break;
3049 
3050         case ECONNABORTED:
3051                 ret = err;
3052                 goto out;
3053 
3054         case EINVAL:
3055         case ELOOP:
3056                 r = libscf_instance_get_fmri(inst, &fmri);
3057                 switch (r) {
3058                 case 0:
3059                         break;
3060 
3061                 case ECONNABORTED:
3062                         ret = ECONNABORTED;
3063                         goto out;
3064 
3065                 case ECANCELED:
3066                         ret = 0;
3067                         goto out;
3068 
3069                 default:
3070                         bad_error("libscf_instance_get_fmri", r);
3071                 }
3072 
3073                 if (err == EINVAL) {
3074                         log_error(LOG_ERR, "Transitioning %s "
3075                             "to maintenance due to misconfiguration.\n",
3076                             fmri ? fmri : "?");
3077                         vertex_send_event(v,
3078                             RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY);
3079                 } else {
3080                         handle_cycle(fmri, path);
3081                         vertex_send_event(v,
3082                             RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE);
3083                 }
3084                 startd_free(fmri, max_scf_fmri_size);
3085                 ret = 0;
3086                 goto out;
3087 
3088         default:
3089                 bad_error("set_dependencies", err);
3090         }
3091 
3092         if (milestone > MILESTONE_NONE) {
3093                 boolean_t aborted = B_FALSE;
3094 
3095                 for (e = uu_list_first(old_deps);
3096                     e != NULL;
3097                     e = uu_list_next(old_deps, e)) {
3098                         vv = e->ge_vertex;
3099 
3100                         if (vertex_unref(vv) == VERTEX_INUSE &&
3101                             eval_subgraph(vv, h) == ECONNABORTED)
3102                                 aborted = B_TRUE;
3103                 }
3104 
3105                 for (e = uu_list_first(v->gv_dependencies);
3106                     e != NULL;
3107                     e = uu_list_next(v->gv_dependencies, e)) {
3108                         if (eval_subgraph(e->ge_vertex, h) ==
3109                             ECONNABORTED)
3110                                 aborted = B_TRUE;
3111                 }
3112 
3113                 if (aborted) {
3114                         ret = ECONNABORTED;
3115                         goto out;
3116                 }
3117         }
3118 
3119         graph_start_if_satisfied(v);
3120 
3121         ret = 0;
3122 
3123 out:
3124         if (milestone > MILESTONE_NONE) {
3125                 void *cookie = NULL;
3126 
3127                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL)
3128                         startd_free(e, sizeof (*e));
3129 
3130                 uu_list_destroy(old_deps);
3131         }
3132 
3133         return (ret);
3134 }
3135 
3136 /*
3137  * Set up v according to inst.  That is, make sure it depends on its
3138  * restarter and set up its dependencies.  Send the ADD_INSTANCE command to
3139  * the restarter, and send ENABLE or DISABLE as appropriate.
3140  *
3141  * Returns 0 on success, ECONNABORTED on repository disconnection, or
3142  * ECANCELED if inst is deleted.
3143  */
3144 static int
3145 configure_vertex(graph_vertex_t *v, scf_instance_t *inst)
3146 {
3147         scf_handle_t *h;
3148         scf_propertygroup_t *pg;
3149         scf_snapshot_t *snap;
3150         char *restarter_fmri = startd_alloc(max_scf_value_size);
3151         int enabled, enabled_ovr;
3152         int err;
3153         int *path;
3154         int deathrow;
3155         int32_t tset;
3156 
3157         restarter_fmri[0] = '\0';
3158 
3159         assert(MUTEX_HELD(&dgraph_lock));
3160         assert(v->gv_type == GVT_INST);
3161         assert((v->gv_flags & GV_CONFIGURED) == 0);
3162 
3163         /* GV_INSUBGRAPH should already be set properly. */
3164         assert(should_be_in_subgraph(v) ==
3165             ((v->gv_flags & GV_INSUBGRAPH) != 0));
3166 
3167         /*
3168          * If the instance fmri is in the deathrow list then set the
3169          * GV_DEATHROW flag on the vertex and create and set to true the
3170          * SCF_PROPERTY_DEATHROW boolean property in the non-persistent
3171          * repository for this instance fmri.
3172          */
3173         if ((v->gv_flags & GV_DEATHROW) ||
3174             (is_fmri_in_deathrow(v->gv_name) == B_TRUE)) {
3175                 if ((v->gv_flags & GV_DEATHROW) == 0) {
3176                         /*
3177                          * Set flag GV_DEATHROW, create and set to true
3178                          * the SCF_PROPERTY_DEATHROW property in the
3179                          * non-persistent repository for this instance fmri.
3180                          */
3181                         v->gv_flags |= GV_DEATHROW;
3182 
3183                         switch (err = libscf_set_deathrow(inst, 1)) {
3184                         case 0:
3185                                 break;
3186 
3187                         case ECONNABORTED:
3188                         case ECANCELED:
3189                                 startd_free(restarter_fmri, max_scf_value_size);
3190                                 return (err);
3191 
3192                         case EROFS:
3193                                 log_error(LOG_WARNING, "Could not set %s/%s "
3194                                     "for deathrow %s: %s.\n",
3195                                     SCF_PG_DEATHROW, SCF_PROPERTY_DEATHROW,
3196                                     v->gv_name, strerror(err));
3197                                 break;
3198 
3199                         case EPERM:
3200                                 uu_die("Permission denied.\n");
3201                                 /* NOTREACHED */
3202 
3203                         default:
3204                                 bad_error("libscf_set_deathrow", err);
3205                         }
3206                         log_framework(LOG_DEBUG, "Deathrow, graph set %s.\n",
3207                             v->gv_name);
3208                 }
3209                 startd_free(restarter_fmri, max_scf_value_size);
3210                 return (0);
3211         }
3212 
3213         h = scf_instance_handle(inst);
3214 
3215         /*
3216          * Using a temporary deathrow boolean property, set through
3217          * libscf_set_deathrow(), only for fmris on deathrow, is necessary
3218          * because deathrow_fini() may already have been called, and in case
3219          * of a refresh, GV_DEATHROW may need to be set again.
3220          * libscf_get_deathrow() sets deathrow to 1 only if this instance
3221          * has a temporary boolean property named 'deathrow' valued true
3222          * in a property group 'deathrow', -1 or 0 in all other cases.
3223          */
3224         err = libscf_get_deathrow(h, inst, &deathrow);
3225         switch (err) {
3226         case 0:
3227                 break;
3228 
3229         case ECONNABORTED:
3230         case ECANCELED:
3231                 startd_free(restarter_fmri, max_scf_value_size);
3232                 return (err);
3233 
3234         default:
3235                 bad_error("libscf_get_deathrow", err);
3236         }
3237 
3238         if (deathrow == 1) {
3239                 v->gv_flags |= GV_DEATHROW;
3240                 startd_free(restarter_fmri, max_scf_value_size);
3241                 return (0);
3242         }
3243 
3244         log_framework(LOG_DEBUG, "Graph adding %s.\n", v->gv_name);
3245 
3246         /*
3247          * If the instance does not have a restarter property group,
3248          * initialize its state to uninitialized/none, in case the restarter
3249          * is not enabled.
3250          */
3251         pg = safe_scf_pg_create(h);
3252 
3253         if (scf_instance_get_pg(inst, SCF_PG_RESTARTER, pg) != 0) {
3254                 instance_data_t idata;
3255                 uint_t count = 0, msecs = ALLOC_DELAY;
3256 
3257                 switch (scf_error()) {
3258                 case SCF_ERROR_NOT_FOUND:
3259                         break;
3260 
3261                 case SCF_ERROR_CONNECTION_BROKEN:
3262                 default:
3263                         scf_pg_destroy(pg);
3264                         startd_free(restarter_fmri, max_scf_value_size);
3265                         return (ECONNABORTED);
3266 
3267                 case SCF_ERROR_DELETED:
3268                         scf_pg_destroy(pg);
3269                         startd_free(restarter_fmri, max_scf_value_size);
3270                         return (ECANCELED);
3271 
3272                 case SCF_ERROR_NOT_SET:
3273                         bad_error("scf_instance_get_pg", scf_error());
3274                 }
3275 
3276                 switch (err = libscf_instance_get_fmri(inst,
3277                     (char **)&idata.i_fmri)) {
3278                 case 0:
3279                         break;
3280 
3281                 case ECONNABORTED:
3282                 case ECANCELED:
3283                         scf_pg_destroy(pg);
3284                         startd_free(restarter_fmri, max_scf_value_size);
3285                         return (err);
3286 
3287                 default:
3288                         bad_error("libscf_instance_get_fmri", err);
3289                 }
3290 
3291                 idata.i_state = RESTARTER_STATE_NONE;
3292                 idata.i_next_state = RESTARTER_STATE_NONE;
3293 
3294 init_state:
3295                 switch (err = _restarter_commit_states(h, &idata,
3296                     RESTARTER_STATE_UNINIT, RESTARTER_STATE_NONE,
3297                     restarter_get_str_short(restarter_str_insert_in_graph))) {
3298                 case 0:
3299                         break;
3300 
3301                 case ENOMEM:
3302                         ++count;
3303                         if (count < ALLOC_RETRY) {
3304                                 (void) poll(NULL, 0, msecs);
3305                                 msecs *= ALLOC_DELAY_MULT;
3306                                 goto init_state;
3307                         }
3308 
3309                         uu_die("Insufficient memory.\n");
3310                         /* NOTREACHED */
3311 
3312                 case ECONNABORTED:
3313                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3314                         scf_pg_destroy(pg);
3315                         startd_free(restarter_fmri, max_scf_value_size);
3316                         return (ECONNABORTED);
3317 
3318                 case ENOENT:
3319                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3320                         scf_pg_destroy(pg);
3321                         startd_free(restarter_fmri, max_scf_value_size);
3322                         return (ECANCELED);
3323 
3324                 case EPERM:
3325                 case EACCES:
3326                 case EROFS:
3327                         log_error(LOG_NOTICE, "Could not initialize state for "
3328                             "%s: %s.\n", idata.i_fmri, strerror(err));
3329                         break;
3330 
3331                 case EINVAL:
3332                 default:
3333                         bad_error("_restarter_commit_states", err);
3334                 }
3335 
3336                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3337         }
3338 
3339         scf_pg_destroy(pg);
3340 
3341         if (milestone != NULL) {
3342                 /*
3343                  * Make sure the enable-override is set properly before we
3344                  * read whether we should be enabled.
3345                  */
3346                 if (milestone == MILESTONE_NONE ||
3347                     !(v->gv_flags & GV_INSUBGRAPH)) {
3348                         /*
3349                          * This might seem unjustified after the milestone
3350                          * transition has completed (non_subgraph_svcs == 0),
3351                          * but it's important because when we boot to
3352                          * a milestone, we set the milestone before populating
3353                          * the graph, and all of the new non-subgraph services
3354                          * need to be disabled here.
3355                          */
3356                         switch (err = libscf_set_enable_ovr(inst, 0)) {
3357                         case 0:
3358                                 break;
3359 
3360                         case ECONNABORTED:
3361                         case ECANCELED:
3362                                 startd_free(restarter_fmri, max_scf_value_size);
3363                                 return (err);
3364 
3365                         case EROFS:
3366                                 log_error(LOG_WARNING,
3367                                     "Could not set %s/%s for %s: %s.\n",
3368                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
3369                                     v->gv_name, strerror(err));
3370                                 break;
3371 
3372                         case EPERM:
3373                                 uu_die("Permission denied.\n");
3374                                 /* NOTREACHED */
3375 
3376                         default:
3377                                 bad_error("libscf_set_enable_ovr", err);
3378                         }
3379                 } else {
3380                         assert(v->gv_flags & GV_INSUBGRAPH);
3381                         switch (err = libscf_delete_enable_ovr(inst)) {
3382                         case 0:
3383                                 break;
3384 
3385                         case ECONNABORTED:
3386                         case ECANCELED:
3387                                 startd_free(restarter_fmri, max_scf_value_size);
3388                                 return (err);
3389 
3390                         case EPERM:
3391                                 uu_die("Permission denied.\n");
3392                                 /* NOTREACHED */
3393 
3394                         default:
3395                                 bad_error("libscf_delete_enable_ovr", err);
3396                         }
3397                 }
3398         }
3399 
3400         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
3401             &enabled_ovr, &restarter_fmri);
3402         switch (err) {
3403         case 0:
3404                 break;
3405 
3406         case ECONNABORTED:
3407         case ECANCELED:
3408                 startd_free(restarter_fmri, max_scf_value_size);
3409                 return (err);
3410 
3411         case ENOENT:
3412                 log_framework(LOG_DEBUG,
3413                     "Ignoring %s because it has no general property group.\n",
3414                     v->gv_name);
3415                 startd_free(restarter_fmri, max_scf_value_size);
3416                 return (0);
3417 
3418         default:
3419                 bad_error("libscf_get_basic_instance_data", err);
3420         }
3421 
3422         if ((tset = libscf_get_stn_tset(inst)) == -1) {
3423                 log_framework(LOG_WARNING,
3424                     "Failed to get notification parameters for %s: %s\n",
3425                     v->gv_name, scf_strerror(scf_error()));
3426                 v->gv_stn_tset = 0;
3427         } else {
3428                 v->gv_stn_tset = tset;
3429         }
3430         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
3431                 stn_global = v->gv_stn_tset;
3432 
3433         if (enabled == -1) {
3434                 startd_free(restarter_fmri, max_scf_value_size);
3435                 return (0);
3436         }
3437 
3438         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
3439             (enabled ? GV_ENBLD_NOOVR : 0);
3440 
3441         if (enabled_ovr != -1)
3442                 enabled = enabled_ovr;
3443 
3444         v->gv_state = RESTARTER_STATE_UNINIT;
3445 
3446         snap = libscf_get_or_make_running_snapshot(inst, v->gv_name, B_TRUE);
3447         scf_snapshot_destroy(snap);
3448 
3449         /* Set up the restarter. (Sends _ADD_INSTANCE on success.) */
3450         err = graph_change_restarter(v, restarter_fmri, h, &path);
3451         if (err != 0) {
3452                 instance_data_t idata;
3453                 uint_t count = 0, msecs = ALLOC_DELAY;
3454                 restarter_str_t reason;
3455 
3456                 if (err == ECONNABORTED) {
3457                         startd_free(restarter_fmri, max_scf_value_size);
3458                         return (err);
3459                 }
3460 
3461                 assert(err == EINVAL || err == ELOOP);
3462 
3463                 if (err == EINVAL) {
3464                         log_framework(LOG_ERR, emsg_invalid_restarter,
3465                             v->gv_name, restarter_fmri);
3466                         reason = restarter_str_invalid_restarter;
3467                 } else {
3468                         handle_cycle(v->gv_name, path);
3469                         reason = restarter_str_dependency_cycle;
3470                 }
3471 
3472                 startd_free(restarter_fmri, max_scf_value_size);
3473 
3474                 /*
3475                  * We didn't register the instance with the restarter, so we
3476                  * must set maintenance mode ourselves.
3477                  */
3478                 err = libscf_instance_get_fmri(inst, (char **)&idata.i_fmri);
3479                 if (err != 0) {
3480                         assert(err == ECONNABORTED || err == ECANCELED);
3481                         return (err);
3482                 }
3483 
3484                 idata.i_state = RESTARTER_STATE_NONE;
3485                 idata.i_next_state = RESTARTER_STATE_NONE;
3486 
3487 set_maint:
3488                 switch (err = _restarter_commit_states(h, &idata,
3489                     RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE,
3490                     restarter_get_str_short(reason))) {
3491                 case 0:
3492                         break;
3493 
3494                 case ENOMEM:
3495                         ++count;
3496                         if (count < ALLOC_RETRY) {
3497                                 (void) poll(NULL, 0, msecs);
3498                                 msecs *= ALLOC_DELAY_MULT;
3499                                 goto set_maint;
3500                         }
3501 
3502                         uu_die("Insufficient memory.\n");
3503                         /* NOTREACHED */
3504 
3505                 case ECONNABORTED:
3506                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3507                         return (ECONNABORTED);
3508 
3509                 case ENOENT:
3510                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3511                         return (ECANCELED);
3512 
3513                 case EPERM:
3514                 case EACCES:
3515                 case EROFS:
3516                         log_error(LOG_NOTICE, "Could not initialize state for "
3517                             "%s: %s.\n", idata.i_fmri, strerror(err));
3518                         break;
3519 
3520                 case EINVAL:
3521                 default:
3522                         bad_error("_restarter_commit_states", err);
3523                 }
3524 
3525                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3526 
3527                 v->gv_state = RESTARTER_STATE_MAINT;
3528 
3529                 goto out;
3530         }
3531         startd_free(restarter_fmri, max_scf_value_size);
3532 
3533         /* Add all the other dependencies. */
3534         err = refresh_vertex(v, inst);
3535         if (err != 0) {
3536                 assert(err == ECONNABORTED);
3537                 return (err);
3538         }
3539 
3540 out:
3541         v->gv_flags |= GV_CONFIGURED;
3542 
3543         graph_enable_by_vertex(v, enabled, 0);
3544 
3545         return (0);
3546 }
3547 
3548 
3549 static void
3550 kill_user_procs(void)
3551 {
3552         (void) fputs("svc.startd: Killing user processes.\n", stdout);
3553 
3554         /*
3555          * Despite its name, killall's role is to get select user processes--
3556          * basically those representing terminal-based logins-- to die.  Victims
3557          * are located by killall in the utmp database.  Since these are most
3558          * often shell based logins, and many shells mask SIGTERM (but are
3559          * responsive to SIGHUP) we first HUP and then shortly thereafter
3560          * kill -9.
3561          */
3562         (void) fork_with_timeout("/usr/sbin/killall HUP", 1, 5);
3563         (void) fork_with_timeout("/usr/sbin/killall KILL", 1, 5);
3564 
3565         /*
3566          * Note the selection of user id's 0, 1 and 15, subsequently
3567          * inverted by -v.  15 is reserved for dladmd.  Yes, this is a
3568          * kludge-- a better policy is needed.
3569          *
3570          * Note that fork_with_timeout will only wait out the 1 second
3571          * "grace time" if pkill actually returns 0.  So if there are
3572          * no matches, this will run to completion much more quickly.
3573          */
3574         (void) fork_with_timeout("/usr/bin/pkill -TERM -v -u 0,1,15", 1, 5);
3575         (void) fork_with_timeout("/usr/bin/pkill -KILL -v -u 0,1,15", 1, 5);
3576 }
3577 
3578 static void
3579 do_uadmin(void)
3580 {
3581         const char * const resetting = "/etc/svc/volatile/resetting";
3582         int fd;
3583         struct statvfs vfs;
3584         time_t now;
3585         struct tm nowtm;
3586         char down_buf[256], time_buf[256];
3587         uintptr_t mdep;
3588 #if defined(__i386)
3589         grub_boot_args_t fbarg;
3590 #endif  /* __i386 */
3591 
3592         mdep = NULL;
3593         fd = creat(resetting, 0777);
3594         if (fd >= 0)
3595                 startd_close(fd);
3596         else
3597                 uu_warn("Could not create \"%s\"", resetting);
3598 
3599         /* Kill dhcpagent if we're not using nfs for root */
3600         if ((statvfs("/", &vfs) == 0) &&
3601             (strncmp(vfs.f_basetype, "nfs", sizeof ("nfs") - 1) != 0))
3602                 fork_with_timeout("/usr/bin/pkill -x -u 0 dhcpagent", 0, 5);
3603 
3604         /*
3605          * Call sync(2) now, before we kill off user processes.  This takes
3606          * advantage of the several seconds of pause we have before the
3607          * killalls are done.  Time we can make good use of to get pages
3608          * moving out to disk.
3609          *
3610          * Inside non-global zones, we don't bother, and it's better not to
3611          * anyway, since sync(2) can have system-wide impact.
3612          */
3613         if (getzoneid() == 0)
3614                 sync();
3615 
3616         kill_user_procs();
3617 
3618         /*
3619          * Note that this must come after the killing of user procs, since
3620          * killall relies on utmpx, and this command affects the contents of
3621          * said file.
3622          */
3623         if (access("/usr/lib/acct/closewtmp", X_OK) == 0)
3624                 fork_with_timeout("/usr/lib/acct/closewtmp", 0, 5);
3625 
3626         /*
3627          * For patches which may be installed as the system is shutting
3628          * down, we need to ensure, one more time, that the boot archive
3629          * really is up to date.
3630          */
3631         if (getzoneid() == 0 && access("/usr/sbin/bootadm", X_OK) == 0)
3632                 fork_with_timeout("/usr/sbin/bootadm -ea update_all", 0, 3600);
3633 
3634         /*
3635          * Right now, fast reboot is supported only on i386.
3636          * scf_is_fastboot_default() should take care of it.
3637          * If somehow we got there on unsupported platform -
3638          * print warning and fall back to regular reboot.
3639          */
3640         if (halting == AD_FASTREBOOT) {
3641 #if defined(__i386)
3642                 int rc;
3643 
3644                 if ((rc = grub_get_boot_args(&fbarg, NULL,
3645                     GRUB_ENTRY_DEFAULT)) == 0) {
3646                         mdep = (uintptr_t)&fbarg.gba_bootargs;
3647                 } else {
3648                         /*
3649                          * Failed to read GRUB menu, fall back to normal reboot
3650                          */
3651                         halting = AD_BOOT;
3652                         uu_warn("Failed to process GRUB menu entry "
3653                             "for fast reboot.\n\t%s\n"
3654                             "Falling back to regular reboot.\n",
3655                             grub_strerror(rc));
3656                 }
3657 #else   /* __i386 */
3658                 halting = AD_BOOT;
3659                 uu_warn("Fast reboot configured, but not supported by "
3660                     "this ISA\n");
3661 #endif  /* __i386 */
3662         }
3663 
3664         fork_with_timeout("/sbin/umountall -l", 0, 5);
3665         fork_with_timeout("/sbin/umount /tmp /var/adm /var/run /var "
3666             ">/dev/null 2>&1", 0, 5);
3667 
3668         /*
3669          * Try to get to consistency for whatever UFS filesystems are left.
3670          * This is pretty expensive, so we save it for the end in the hopes of
3671          * minimizing what it must do.  The other option would be to start in
3672          * parallel with the killall's, but lockfs tends to throw out much more
3673          * than is needed, and so subsequent commands (like umountall) take a
3674          * long time to get going again.
3675          *
3676          * Inside of zones, we don't bother, since we're not about to terminate
3677          * the whole OS instance.
3678          *
3679          * On systems using only ZFS, this call to lockfs -fa is a no-op.
3680          */
3681         if (getzoneid() == 0) {
3682                 if (access("/usr/sbin/lockfs", X_OK) == 0)
3683                         fork_with_timeout("/usr/sbin/lockfs -fa", 0, 30);
3684 
3685                 sync(); /* once more, with feeling */
3686         }
3687 
3688         fork_with_timeout("/sbin/umount /usr >/dev/null 2>&1", 0, 5);
3689 
3690         /*
3691          * Construct and emit the last words from userland:
3692          * "<timestamp> The system is down.  Shutdown took <N> seconds."
3693          *
3694          * Normally we'd use syslog, but with /var and other things
3695          * potentially gone, try to minimize the external dependencies.
3696          */
3697         now = time(NULL);
3698         (void) localtime_r(&now, &nowtm);
3699 
3700         if (strftime(down_buf, sizeof (down_buf),
3701             "%b %e %T The system is down.", &nowtm) == 0) {
3702                 (void) strlcpy(down_buf, "The system is down.",
3703                     sizeof (down_buf));
3704         }
3705 
3706         if (halting_time != 0 && halting_time <= now) {
3707                 (void) snprintf(time_buf, sizeof (time_buf),
3708                     "  Shutdown took %lu seconds.", now - halting_time);
3709         } else {
3710                 time_buf[0] = '\0';
3711         }
3712         (void) printf("%s%s\n", down_buf, time_buf);
3713 
3714         (void) uadmin(A_SHUTDOWN, halting, mdep);
3715         uu_warn("uadmin() failed");
3716 
3717 #if defined(__i386)
3718         /* uadmin fail, cleanup grub_boot_args */
3719         if (halting == AD_FASTREBOOT)
3720                 grub_cleanup_boot_args(&fbarg);
3721 #endif  /* __i386 */
3722 
3723         if (remove(resetting) != 0 && errno != ENOENT)
3724                 uu_warn("Could not remove \"%s\"", resetting);
3725 }
3726 
3727 /*
3728  * If any of the up_svcs[] are online or satisfiable, return true.  If they are
3729  * all missing, disabled, in maintenance, or unsatisfiable, return false.
3730  */
3731 boolean_t
3732 can_come_up(void)
3733 {
3734         int i;
3735 
3736         assert(MUTEX_HELD(&dgraph_lock));
3737 
3738         /*
3739          * If we are booting to single user (boot -s),
3740          * SCF_MILESTONE_SINGLE_USER is needed to come up because startd
3741          * spawns sulogin after single-user is online (see specials.c).
3742          */
3743         i = (booting_to_single_user ? 0 : 1);
3744 
3745         for (; up_svcs[i] != NULL; ++i) {
3746                 if (up_svcs_p[i] == NULL) {
3747                         up_svcs_p[i] = vertex_get_by_name(up_svcs[i]);
3748 
3749                         if (up_svcs_p[i] == NULL)
3750                                 continue;
3751                 }
3752 
3753                 /*
3754                  * Ignore unconfigured services (the ones that have been
3755                  * mentioned in a dependency from other services, but do
3756                  * not exist in the repository).  Services which exist
3757                  * in the repository but don't have general/enabled
3758                  * property will be also ignored.
3759                  */
3760                 if (!(up_svcs_p[i]->gv_flags & GV_CONFIGURED))
3761                         continue;
3762 
3763                 switch (up_svcs_p[i]->gv_state) {
3764                 case RESTARTER_STATE_ONLINE:
3765                 case RESTARTER_STATE_DEGRADED:
3766                         /*
3767                          * Deactivate verbose boot once a login service has been
3768                          * reached.
3769                          */
3770                         st->st_log_login_reached = 1;
3771                         /*FALLTHROUGH*/
3772                 case RESTARTER_STATE_UNINIT:
3773                         return (B_TRUE);
3774 
3775                 case RESTARTER_STATE_OFFLINE:
3776                         if (instance_satisfied(up_svcs_p[i], B_TRUE) != -1)
3777                                 return (B_TRUE);
3778                         log_framework(LOG_DEBUG,
3779                             "can_come_up(): %s is unsatisfiable.\n",
3780                             up_svcs_p[i]->gv_name);
3781                         continue;
3782 
3783                 case RESTARTER_STATE_DISABLED:
3784                 case RESTARTER_STATE_MAINT:
3785                         log_framework(LOG_DEBUG,
3786                             "can_come_up(): %s is in state %s.\n",
3787                             up_svcs_p[i]->gv_name,
3788                             instance_state_str[up_svcs_p[i]->gv_state]);
3789                         continue;
3790 
3791                 default:
3792 #ifndef NDEBUG
3793                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
3794                             __FILE__, __LINE__, up_svcs_p[i]->gv_state);
3795 #endif
3796                         abort();
3797                 }
3798         }
3799 
3800         /*
3801          * In the seed repository, console-login is unsatisfiable because
3802          * services are missing.  To behave correctly in that case we don't want
3803          * to return false until manifest-import is online.
3804          */
3805 
3806         if (manifest_import_p == NULL) {
3807                 manifest_import_p = vertex_get_by_name(manifest_import);
3808 
3809                 if (manifest_import_p == NULL)
3810                         return (B_FALSE);
3811         }
3812 
3813         switch (manifest_import_p->gv_state) {
3814         case RESTARTER_STATE_ONLINE:
3815         case RESTARTER_STATE_DEGRADED:
3816         case RESTARTER_STATE_DISABLED:
3817         case RESTARTER_STATE_MAINT:
3818                 break;
3819 
3820         case RESTARTER_STATE_OFFLINE:
3821                 if (instance_satisfied(manifest_import_p, B_TRUE) == -1)
3822                         break;
3823                 /* FALLTHROUGH */
3824 
3825         case RESTARTER_STATE_UNINIT:
3826                 return (B_TRUE);
3827         }
3828 
3829         return (B_FALSE);
3830 }
3831 
3832 /*
3833  * Runs sulogin.  Returns
3834  *   0 - success
3835  *   EALREADY - sulogin is already running
3836  *   EBUSY - console-login is running
3837  */
3838 static int
3839 run_sulogin(const char *msg)
3840 {
3841         graph_vertex_t *v;
3842 
3843         assert(MUTEX_HELD(&dgraph_lock));
3844 
3845         if (sulogin_running)
3846                 return (EALREADY);
3847 
3848         v = vertex_get_by_name(console_login_fmri);
3849         if (v != NULL && inst_running(v))
3850                 return (EBUSY);
3851 
3852         sulogin_running = B_TRUE;
3853 
3854         MUTEX_UNLOCK(&dgraph_lock);
3855 
3856         fork_sulogin(B_FALSE, msg);
3857 
3858         MUTEX_LOCK(&dgraph_lock);
3859 
3860         sulogin_running = B_FALSE;
3861 
3862         if (console_login_ready) {
3863                 v = vertex_get_by_name(console_login_fmri);
3864 
3865                 if (v != NULL && v->gv_state == RESTARTER_STATE_OFFLINE) {
3866                         if (v->gv_start_f == NULL)
3867                                 vertex_send_event(v,
3868                                     RESTARTER_EVENT_TYPE_START);
3869                         else
3870                                 v->gv_start_f(v);
3871                 }
3872 
3873                 console_login_ready = B_FALSE;
3874         }
3875 
3876         return (0);
3877 }
3878 
3879 /*
3880  * The sulogin thread runs sulogin while can_come_up() is false.  run_sulogin()
3881  * keeps sulogin from stepping on console-login's toes.
3882  */
3883 /* ARGSUSED */
3884 static void *
3885 sulogin_thread(void *unused)
3886 {
3887         MUTEX_LOCK(&dgraph_lock);
3888 
3889         assert(sulogin_thread_running);
3890 
3891         do {
3892                 (void) run_sulogin("Console login service(s) cannot run\n");
3893         } while (!can_come_up());
3894 
3895         sulogin_thread_running = B_FALSE;
3896         MUTEX_UNLOCK(&dgraph_lock);
3897 
3898         return (NULL);
3899 }
3900 
3901 /* ARGSUSED */
3902 void *
3903 single_user_thread(void *unused)
3904 {
3905         uint_t left;
3906         scf_handle_t *h;
3907         scf_instance_t *inst;
3908         scf_property_t *prop;
3909         scf_value_t *val;
3910         const char *msg;
3911         char *buf;
3912         int r;
3913 
3914         MUTEX_LOCK(&single_user_thread_lock);
3915         single_user_thread_count++;
3916 
3917         if (!booting_to_single_user)
3918                 kill_user_procs();
3919 
3920         if (go_single_user_mode || booting_to_single_user) {
3921                 msg = "SINGLE USER MODE\n";
3922         } else {
3923                 assert(go_to_level1);
3924 
3925                 fork_rc_script('1', "start", B_TRUE);
3926 
3927                 uu_warn("The system is ready for administration.\n");
3928 
3929                 msg = "";
3930         }
3931 
3932         MUTEX_UNLOCK(&single_user_thread_lock);
3933 
3934         for (;;) {
3935                 MUTEX_LOCK(&dgraph_lock);
3936                 r = run_sulogin(msg);
3937                 MUTEX_UNLOCK(&dgraph_lock);
3938                 if (r == 0)
3939                         break;
3940 
3941                 assert(r == EALREADY || r == EBUSY);
3942 
3943                 left = 3;
3944                 while (left > 0)
3945                         left = sleep(left);
3946         }
3947 
3948         MUTEX_LOCK(&single_user_thread_lock);
3949 
3950         /*
3951          * If another single user thread has started, let it finish changing
3952          * the run level.
3953          */
3954         if (single_user_thread_count > 1) {
3955                 single_user_thread_count--;
3956                 MUTEX_UNLOCK(&single_user_thread_lock);
3957                 return (NULL);
3958         }
3959 
3960         h = libscf_handle_create_bound_loop();
3961         inst = scf_instance_create(h);
3962         prop = safe_scf_property_create(h);
3963         val = safe_scf_value_create(h);
3964         buf = startd_alloc(max_scf_fmri_size);
3965 
3966 lookup:
3967         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
3968             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
3969                 switch (scf_error()) {
3970                 case SCF_ERROR_NOT_FOUND:
3971                         r = libscf_create_self(h);
3972                         if (r == 0)
3973                                 goto lookup;
3974                         assert(r == ECONNABORTED);
3975                         /* FALLTHROUGH */
3976 
3977                 case SCF_ERROR_CONNECTION_BROKEN:
3978                         libscf_handle_rebind(h);
3979                         goto lookup;
3980 
3981                 case SCF_ERROR_INVALID_ARGUMENT:
3982                 case SCF_ERROR_CONSTRAINT_VIOLATED:
3983                 case SCF_ERROR_NOT_BOUND:
3984                 case SCF_ERROR_HANDLE_MISMATCH:
3985                 default:
3986                         bad_error("scf_handle_decode_fmri", scf_error());
3987                 }
3988         }
3989 
3990         MUTEX_LOCK(&dgraph_lock);
3991 
3992         r = scf_instance_delete_prop(inst, SCF_PG_OPTIONS_OVR,
3993             SCF_PROPERTY_MILESTONE);
3994         switch (r) {
3995         case 0:
3996         case ECANCELED:
3997                 break;
3998 
3999         case ECONNABORTED:
4000                 MUTEX_UNLOCK(&dgraph_lock);
4001                 libscf_handle_rebind(h);
4002                 goto lookup;
4003 
4004         case EPERM:
4005         case EACCES:
4006         case EROFS:
4007                 log_error(LOG_WARNING, "Could not clear temporary milestone: "
4008                     "%s.\n", strerror(r));
4009                 break;
4010 
4011         default:
4012                 bad_error("scf_instance_delete_prop", r);
4013         }
4014 
4015         MUTEX_UNLOCK(&dgraph_lock);
4016 
4017         r = libscf_get_milestone(inst, prop, val, buf, max_scf_fmri_size);
4018         switch (r) {
4019         case ECANCELED:
4020         case ENOENT:
4021         case EINVAL:
4022                 (void) strcpy(buf, "all");
4023                 /* FALLTHROUGH */
4024 
4025         case 0:
4026                 uu_warn("Returning to milestone %s.\n", buf);
4027                 break;
4028 
4029         case ECONNABORTED:
4030                 libscf_handle_rebind(h);
4031                 goto lookup;
4032 
4033         default:
4034                 bad_error("libscf_get_milestone", r);
4035         }
4036 
4037         r = dgraph_set_milestone(buf, h, B_FALSE);
4038         switch (r) {
4039         case 0:
4040         case ECONNRESET:
4041         case EALREADY:
4042         case EINVAL:
4043         case ENOENT:
4044                 break;
4045 
4046         default:
4047                 bad_error("dgraph_set_milestone", r);
4048         }
4049 
4050         /*
4051          * See graph_runlevel_changed().
4052          */
4053         MUTEX_LOCK(&dgraph_lock);
4054         utmpx_set_runlevel(target_milestone_as_runlevel(), 'S', B_TRUE);
4055         MUTEX_UNLOCK(&dgraph_lock);
4056 
4057         startd_free(buf, max_scf_fmri_size);
4058         scf_value_destroy(val);
4059         scf_property_destroy(prop);
4060         scf_instance_destroy(inst);
4061         scf_handle_destroy(h);
4062 
4063         /*
4064          * We'll give ourselves 3 seconds to respond to all of the enablings
4065          * that setting the milestone should have created before checking
4066          * whether to run sulogin.
4067          */
4068         left = 3;
4069         while (left > 0)
4070                 left = sleep(left);
4071 
4072         MUTEX_LOCK(&dgraph_lock);
4073         /*
4074          * Clearing these variables will allow the sulogin thread to run.  We
4075          * check here in case there aren't any more state updates anytime soon.
4076          */
4077         go_to_level1 = go_single_user_mode = booting_to_single_user = B_FALSE;
4078         if (!sulogin_thread_running && !can_come_up()) {
4079                 (void) startd_thread_create(sulogin_thread, NULL);
4080                 sulogin_thread_running = B_TRUE;
4081         }
4082         MUTEX_UNLOCK(&dgraph_lock);
4083         single_user_thread_count--;
4084         MUTEX_UNLOCK(&single_user_thread_lock);
4085         return (NULL);
4086 }
4087 
4088 
4089 /*
4090  * Dependency graph operations API.  These are handle-independent thread-safe
4091  * graph manipulation functions which are the entry points for the event
4092  * threads below.
4093  */
4094 
4095 /*
4096  * If a configured vertex exists for inst_fmri, return EEXIST.  If no vertex
4097  * exists for inst_fmri, add one.  Then fetch the restarter from inst, make
4098  * this vertex dependent on it, and send _ADD_INSTANCE to the restarter.
4099  * Fetch whether the instance should be enabled from inst and send _ENABLE or
4100  * _DISABLE as appropriate.  Finally rummage through inst's dependency
4101  * property groups and add vertices and edges as appropriate.  If anything
4102  * goes wrong after sending _ADD_INSTANCE, send _ADMIN_MAINT_ON to put the
4103  * instance in maintenance.  Don't send _START or _STOP until we get a state
4104  * update in case we're being restarted and the service is already running.
4105  *
4106  * To support booting to a milestone, we must also make sure all dependencies
4107  * encountered are configured, if they exist in the repository.
4108  *
4109  * Returns 0 on success, ECONNABORTED on repository disconnection, EINVAL if
4110  * inst_fmri is an invalid (or not canonical) FMRI, ECANCELED if inst is
4111  * deleted, or EEXIST if a configured vertex for inst_fmri already exists.
4112  */
4113 int
4114 dgraph_add_instance(const char *inst_fmri, scf_instance_t *inst,
4115     boolean_t lock_graph)
4116 {
4117         graph_vertex_t *v;
4118         int err;
4119 
4120         if (strcmp(inst_fmri, SCF_SERVICE_STARTD) == 0)
4121                 return (0);
4122 
4123         /* Check for a vertex for inst_fmri. */
4124         if (lock_graph) {
4125                 MUTEX_LOCK(&dgraph_lock);
4126         } else {
4127                 assert(MUTEX_HELD(&dgraph_lock));
4128         }
4129 
4130         v = vertex_get_by_name(inst_fmri);
4131 
4132         if (v != NULL) {
4133                 assert(v->gv_type == GVT_INST);
4134 
4135                 if (v->gv_flags & GV_CONFIGURED) {
4136                         if (lock_graph)
4137                                 MUTEX_UNLOCK(&dgraph_lock);
4138                         return (EEXIST);
4139                 }
4140         } else {
4141                 /* Add the vertex. */
4142                 err = graph_insert_vertex_unconfigured(inst_fmri, GVT_INST, 0,
4143                     RERR_NONE, &v);
4144                 if (err != 0) {
4145                         assert(err == EINVAL);
4146                         if (lock_graph)
4147                                 MUTEX_UNLOCK(&dgraph_lock);
4148                         return (EINVAL);
4149                 }
4150         }
4151 
4152         err = configure_vertex(v, inst);
4153 
4154         if (lock_graph)
4155                 MUTEX_UNLOCK(&dgraph_lock);
4156 
4157         return (err);
4158 }
4159 
4160 /*
4161  * Locate the vertex for this property group's instance.  If it doesn't exist
4162  * or is unconfigured, call dgraph_add_instance() & return.  Otherwise fetch
4163  * the restarter for the instance, and if it has changed, send
4164  * _REMOVE_INSTANCE to the old restarter, remove the dependency, make sure the
4165  * new restarter has a vertex, add a new dependency, and send _ADD_INSTANCE to
4166  * the new restarter.  Then fetch whether the instance should be enabled, and
4167  * if it is different from what we had, or if we changed the restarter, send
4168  * the appropriate _ENABLE or _DISABLE command.
4169  *
4170  * Returns 0 on success, ENOTSUP if the pg's parent is not an instance,
4171  * ECONNABORTED on repository disconnection, ECANCELED if the instance is
4172  * deleted, or -1 if the instance's general property group is deleted or if
4173  * its enabled property is misconfigured.
4174  */
4175 static int
4176 dgraph_update_general(scf_propertygroup_t *pg)
4177 {
4178         scf_handle_t *h;
4179         scf_instance_t *inst;
4180         char *fmri;
4181         char *restarter_fmri;
4182         graph_vertex_t *v;
4183         int err;
4184         int enabled, enabled_ovr;
4185         int oldflags;
4186 
4187         /* Find the vertex for this service */
4188         h = scf_pg_handle(pg);
4189 
4190         inst = safe_scf_instance_create(h);
4191 
4192         if (scf_pg_get_parent_instance(pg, inst) != 0) {
4193                 switch (scf_error()) {
4194                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4195                         return (ENOTSUP);
4196 
4197                 case SCF_ERROR_CONNECTION_BROKEN:
4198                 default:
4199                         return (ECONNABORTED);
4200 
4201                 case SCF_ERROR_DELETED:
4202                         return (0);
4203 
4204                 case SCF_ERROR_NOT_SET:
4205                         bad_error("scf_pg_get_parent_instance", scf_error());
4206                 }
4207         }
4208 
4209         err = libscf_instance_get_fmri(inst, &fmri);
4210         switch (err) {
4211         case 0:
4212                 break;
4213 
4214         case ECONNABORTED:
4215                 scf_instance_destroy(inst);
4216                 return (ECONNABORTED);
4217 
4218         case ECANCELED:
4219                 scf_instance_destroy(inst);
4220                 return (0);
4221 
4222         default:
4223                 bad_error("libscf_instance_get_fmri", err);
4224         }
4225 
4226         log_framework(LOG_DEBUG,
4227             "Graph engine: Reloading general properties for %s.\n", fmri);
4228 
4229         MUTEX_LOCK(&dgraph_lock);
4230 
4231         v = vertex_get_by_name(fmri);
4232         if (v == NULL || !(v->gv_flags & GV_CONFIGURED)) {
4233                 /* Will get the up-to-date properties. */
4234                 MUTEX_UNLOCK(&dgraph_lock);
4235                 err = dgraph_add_instance(fmri, inst, B_TRUE);
4236                 startd_free(fmri, max_scf_fmri_size);
4237                 scf_instance_destroy(inst);
4238                 return (err == ECANCELED ? 0 : err);
4239         }
4240 
4241         /* Read enabled & restarter from repository. */
4242         restarter_fmri = startd_alloc(max_scf_value_size);
4243         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
4244             &enabled_ovr, &restarter_fmri);
4245         if (err != 0 || enabled == -1) {
4246                 MUTEX_UNLOCK(&dgraph_lock);
4247                 scf_instance_destroy(inst);
4248                 startd_free(fmri, max_scf_fmri_size);
4249 
4250                 switch (err) {
4251                 case ENOENT:
4252                 case 0:
4253                         startd_free(restarter_fmri, max_scf_value_size);
4254                         return (-1);
4255 
4256                 case ECONNABORTED:
4257                 case ECANCELED:
4258                         startd_free(restarter_fmri, max_scf_value_size);
4259                         return (err);
4260 
4261                 default:
4262                         bad_error("libscf_get_basic_instance_data", err);
4263                 }
4264         }
4265 
4266         oldflags = v->gv_flags;
4267         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
4268             (enabled ? GV_ENBLD_NOOVR : 0);
4269 
4270         if (enabled_ovr != -1)
4271                 enabled = enabled_ovr;
4272 
4273         /*
4274          * If GV_ENBLD_NOOVR has changed, then we need to re-evaluate the
4275          * subgraph.
4276          */
4277         if (milestone > MILESTONE_NONE && v->gv_flags != oldflags)
4278                 (void) eval_subgraph(v, h);
4279 
4280         scf_instance_destroy(inst);
4281 
4282         /* Ignore restarter change for now. */
4283 
4284         startd_free(restarter_fmri, max_scf_value_size);
4285         startd_free(fmri, max_scf_fmri_size);
4286 
4287         /*
4288          * Always send _ENABLE or _DISABLE.  We could avoid this if the
4289          * restarter didn't change and the enabled value didn't change, but
4290          * that's not easy to check and improbable anyway, so we'll just do
4291          * this.
4292          */
4293         graph_enable_by_vertex(v, enabled, 1);
4294 
4295         MUTEX_UNLOCK(&dgraph_lock);
4296 
4297         return (0);
4298 }
4299 
4300 /*
4301  * Delete all of the property group dependencies of v, update inst's running
4302  * snapshot, and add the dependencies in the new snapshot.  If any of the new
4303  * dependencies would create a cycle, send _ADMIN_MAINT_ON.  Otherwise
4304  * reevaluate v's dependencies, send _START or _STOP as appropriate, and do
4305  * the same for v's dependents.
4306  *
4307  * Returns
4308  *   0 - success
4309  *   ECONNABORTED - repository connection broken
4310  *   ECANCELED - inst was deleted
4311  *   EINVAL - inst is invalid (e.g., missing general/enabled)
4312  *   -1 - libscf_snapshots_refresh() failed
4313  */
4314 static int
4315 dgraph_refresh_instance(graph_vertex_t *v, scf_instance_t *inst)
4316 {
4317         int r;
4318         int enabled;
4319         int32_t tset;
4320 
4321         assert(MUTEX_HELD(&dgraph_lock));
4322         assert(v->gv_type == GVT_INST);
4323 
4324         /* Only refresh services with valid general/enabled properties. */
4325         r = libscf_get_basic_instance_data(scf_instance_handle(inst), inst,
4326             v->gv_name, &enabled, NULL, NULL);
4327         switch (r) {
4328         case 0:
4329                 break;
4330 
4331         case ECONNABORTED:
4332         case ECANCELED:
4333                 return (r);
4334 
4335         case ENOENT:
4336                 log_framework(LOG_DEBUG,
4337                     "Ignoring %s because it has no general property group.\n",
4338                     v->gv_name);
4339                 return (EINVAL);
4340 
4341         default:
4342                 bad_error("libscf_get_basic_instance_data", r);
4343         }
4344 
4345         if ((tset = libscf_get_stn_tset(inst)) == -1) {
4346                 log_framework(LOG_WARNING,
4347                     "Failed to get notification parameters for %s: %s\n",
4348                     v->gv_name, scf_strerror(scf_error()));
4349                 tset = 0;
4350         }
4351         v->gv_stn_tset = tset;
4352         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
4353                 stn_global = tset;
4354 
4355         if (enabled == -1)
4356                 return (EINVAL);
4357 
4358         r = libscf_snapshots_refresh(inst, v->gv_name);
4359         if (r != 0) {
4360                 if (r != -1)
4361                         bad_error("libscf_snapshots_refresh", r);
4362 
4363                 /* error logged */
4364                 return (r);
4365         }
4366 
4367         r = refresh_vertex(v, inst);
4368         if (r != 0 && r != ECONNABORTED)
4369                 bad_error("refresh_vertex", r);
4370         return (r);
4371 }
4372 
4373 /*
4374  * Returns true only if none of this service's dependents are 'up' -- online
4375  * or degraded (offline is considered down in this situation). This function
4376  * is somehow similar to is_nonsubgraph_leaf() but works on subtrees.
4377  */
4378 static boolean_t
4379 insubtree_dependents_down(graph_vertex_t *v)
4380 {
4381         graph_vertex_t *vv;
4382         graph_edge_t *e;
4383 
4384         assert(MUTEX_HELD(&dgraph_lock));
4385 
4386         for (e = uu_list_first(v->gv_dependents); e != NULL;
4387             e = uu_list_next(v->gv_dependents, e)) {
4388                 vv = e->ge_vertex;
4389                 if (vv->gv_type == GVT_INST) {
4390                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4391                                 continue;
4392 
4393                         if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4394                                 continue;
4395 
4396                         if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4397                             (vv->gv_state == RESTARTER_STATE_DEGRADED))
4398                                 return (B_FALSE);
4399                 } else {
4400                         /*
4401                          * Skip all excluded and optional_all dependencies
4402                          * and decide whether to offline the service based
4403                          * on restart_on attribute.
4404                          */
4405                         if (is_depgrp_bypassed(vv))
4406                                 continue;
4407 
4408                         /*
4409                          * For dependency groups or service vertices, keep
4410                          * traversing to see if instances are running.
4411                          */
4412                         if (insubtree_dependents_down(vv) == B_FALSE)
4413                                 return (B_FALSE);
4414                 }
4415         }
4416 
4417         return (B_TRUE);
4418 }
4419 
4420 /*
4421  * Returns true only if none of this service's dependents are 'up' -- online,
4422  * degraded, or offline.
4423  */
4424 static int
4425 is_nonsubgraph_leaf(graph_vertex_t *v)
4426 {
4427         graph_vertex_t *vv;
4428         graph_edge_t *e;
4429 
4430         assert(MUTEX_HELD(&dgraph_lock));
4431 
4432         for (e = uu_list_first(v->gv_dependents);
4433             e != NULL;
4434             e = uu_list_next(v->gv_dependents, e)) {
4435 
4436                 vv = e->ge_vertex;
4437                 if (vv->gv_type == GVT_INST) {
4438                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4439                                 continue;
4440 
4441                         if (vv->gv_flags & GV_INSUBGRAPH)
4442                                 continue;
4443 
4444                         if (up_state(vv->gv_state))
4445                                 return (0);
4446                 } else {
4447                         /*
4448                          * For dependency group or service vertices, keep
4449                          * traversing to see if instances are running.
4450                          *
4451                          * We should skip exclude_all dependencies otherwise
4452                          * the vertex will never be considered as a leaf
4453                          * if the dependent is offline. The main reason for
4454                          * this is that disable_nonsubgraph_leaves() skips
4455                          * exclusion dependencies.
4456                          */
4457                         if (vv->gv_type == GVT_GROUP &&
4458                             vv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4459                                 continue;
4460 
4461                         if (!is_nonsubgraph_leaf(vv))
4462                                 return (0);
4463                 }
4464         }
4465 
4466         return (1);
4467 }
4468 
4469 /*
4470  * Disable v temporarily.  Attempt to do this by setting its enabled override
4471  * property in the repository.  If that fails, send a _DISABLE command.
4472  * Returns 0 on success and ECONNABORTED if the repository connection is
4473  * broken.
4474  */
4475 static int
4476 disable_service_temporarily(graph_vertex_t *v, scf_handle_t *h)
4477 {
4478         const char * const emsg = "Could not temporarily disable %s because "
4479             "%s.  Will stop service anyways.  Repository status for the "
4480             "service may be inaccurate.\n";
4481         const char * const emsg_cbroken =
4482             "the repository connection was broken";
4483 
4484         scf_instance_t *inst;
4485         int r;
4486 
4487         inst = scf_instance_create(h);
4488         if (inst == NULL) {
4489                 char buf[100];
4490 
4491                 (void) snprintf(buf, sizeof (buf),
4492                     "scf_instance_create() failed (%s)",
4493                     scf_strerror(scf_error()));
4494                 log_error(LOG_WARNING, emsg, v->gv_name, buf);
4495 
4496                 graph_enable_by_vertex(v, 0, 0);
4497                 return (0);
4498         }
4499 
4500         r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
4501             NULL, NULL, SCF_DECODE_FMRI_EXACT);
4502         if (r != 0) {
4503                 switch (scf_error()) {
4504                 case SCF_ERROR_CONNECTION_BROKEN:
4505                         log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4506                         graph_enable_by_vertex(v, 0, 0);
4507                         return (ECONNABORTED);
4508 
4509                 case SCF_ERROR_NOT_FOUND:
4510                         return (0);
4511 
4512                 case SCF_ERROR_HANDLE_MISMATCH:
4513                 case SCF_ERROR_INVALID_ARGUMENT:
4514                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4515                 case SCF_ERROR_NOT_BOUND:
4516                 default:
4517                         bad_error("scf_handle_decode_fmri",
4518                             scf_error());
4519                 }
4520         }
4521 
4522         r = libscf_set_enable_ovr(inst, 0);
4523         switch (r) {
4524         case 0:
4525                 scf_instance_destroy(inst);
4526                 return (0);
4527 
4528         case ECANCELED:
4529                 scf_instance_destroy(inst);
4530                 return (0);
4531 
4532         case ECONNABORTED:
4533                 log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4534                 graph_enable_by_vertex(v, 0, 0);
4535                 return (ECONNABORTED);
4536 
4537         case EPERM:
4538                 log_error(LOG_WARNING, emsg, v->gv_name,
4539                     "the repository denied permission");
4540                 graph_enable_by_vertex(v, 0, 0);
4541                 return (0);
4542 
4543         case EROFS:
4544                 log_error(LOG_WARNING, emsg, v->gv_name,
4545                     "the repository is read-only");
4546                 graph_enable_by_vertex(v, 0, 0);
4547                 return (0);
4548 
4549         default:
4550                 bad_error("libscf_set_enable_ovr", r);
4551                 /* NOTREACHED */
4552         }
4553 }
4554 
4555 /*
4556  * Of the transitive instance dependencies of v, offline those which are
4557  * in the subtree and which are leaves (i.e., have no dependents which are
4558  * "up").
4559  */
4560 void
4561 offline_subtree_leaves(graph_vertex_t *v, void *arg)
4562 {
4563         assert(MUTEX_HELD(&dgraph_lock));
4564 
4565         /* If v isn't an instance, recurse on its dependencies. */
4566         if (v->gv_type != GVT_INST) {
4567                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4568                 return;
4569         }
4570 
4571         /*
4572          * If v is not in the subtree, so should all of its dependencies,
4573          * so do nothing.
4574          */
4575         if ((v->gv_flags & GV_TOOFFLINE) == 0)
4576                 return;
4577 
4578         /* If v isn't a leaf because it's already down, recurse. */
4579         if (!up_state(v->gv_state)) {
4580                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4581                 return;
4582         }
4583 
4584         /* if v is a leaf, offline it or disable it if it's the last one */
4585         if (insubtree_dependents_down(v) == B_TRUE) {
4586                 if (v->gv_flags & GV_TODISABLE)
4587                         vertex_send_event(v,
4588                             RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
4589                 else
4590                         offline_vertex(v);
4591         }
4592 }
4593 
4594 void
4595 graph_offline_subtree_leaves(graph_vertex_t *v, void *h)
4596 {
4597         graph_walk_dependencies(v, offline_subtree_leaves, (void *)h);
4598 }
4599 
4600 
4601 /*
4602  * Of the transitive instance dependencies of v, disable those which are not
4603  * in the subgraph and which are leaves (i.e., have no dependents which are
4604  * "up").
4605  */
4606 static void
4607 disable_nonsubgraph_leaves(graph_vertex_t *v, void *arg)
4608 {
4609         assert(MUTEX_HELD(&dgraph_lock));
4610 
4611         /*
4612          * We must skip exclusion dependencies because they are allowed to
4613          * complete dependency cycles.  This is correct because A's exclusion
4614          * dependency on B doesn't bear on the order in which they should be
4615          * stopped.  Indeed, the exclusion dependency should guarantee that
4616          * they are never online at the same time.
4617          */
4618         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4619                 return;
4620 
4621         /* If v isn't an instance, recurse on its dependencies. */
4622         if (v->gv_type != GVT_INST)
4623                 goto recurse;
4624 
4625         if ((v->gv_flags & GV_CONFIGURED) == 0)
4626                 /*
4627                  * Unconfigured instances should have no dependencies, but in
4628                  * case they ever get them,
4629                  */
4630                 goto recurse;
4631 
4632         /*
4633          * If v is in the subgraph, so should all of its dependencies, so do
4634          * nothing.
4635          */
4636         if (v->gv_flags & GV_INSUBGRAPH)
4637                 return;
4638 
4639         /* If v isn't a leaf because it's already down, recurse. */
4640         if (!up_state(v->gv_state))
4641                 goto recurse;
4642 
4643         /* If v is disabled but not down yet, be patient. */
4644         if ((v->gv_flags & GV_ENABLED) == 0)
4645                 return;
4646 
4647         /* If v is a leaf, disable it. */
4648         if (is_nonsubgraph_leaf(v))
4649                 (void) disable_service_temporarily(v, (scf_handle_t *)arg);
4650 
4651         return;
4652 
4653 recurse:
4654         graph_walk_dependencies(v, disable_nonsubgraph_leaves, arg);
4655 }
4656 
4657 static int
4658 stn_restarter_state(restarter_instance_state_t rstate)
4659 {
4660         static const struct statemap {
4661                 restarter_instance_state_t restarter_state;
4662                 int scf_state;
4663         } map[] = {
4664                 { RESTARTER_STATE_UNINIT, SCF_STATE_UNINIT },
4665                 { RESTARTER_STATE_MAINT, SCF_STATE_MAINT },
4666                 { RESTARTER_STATE_OFFLINE, SCF_STATE_OFFLINE },
4667                 { RESTARTER_STATE_DISABLED, SCF_STATE_DISABLED },
4668                 { RESTARTER_STATE_ONLINE, SCF_STATE_ONLINE },
4669                 { RESTARTER_STATE_DEGRADED, SCF_STATE_DEGRADED }
4670         };
4671 
4672         int i;
4673 
4674         for (i = 0; i < sizeof (map) / sizeof (map[0]); i++) {
4675                 if (rstate == map[i].restarter_state)
4676                         return (map[i].scf_state);
4677         }
4678 
4679         return (-1);
4680 }
4681 
4682 /*
4683  * State transition counters
4684  * Not incremented atomically - indicative only
4685  */
4686 static uint64_t stev_ct_maint;
4687 static uint64_t stev_ct_hwerr;
4688 static uint64_t stev_ct_service;
4689 static uint64_t stev_ct_global;
4690 static uint64_t stev_ct_noprefs;
4691 static uint64_t stev_ct_from_uninit;
4692 static uint64_t stev_ct_bad_state;
4693 static uint64_t stev_ct_ovr_prefs;
4694 
4695 static void
4696 dgraph_state_transition_notify(graph_vertex_t *v,
4697     restarter_instance_state_t old_state, restarter_str_t reason)
4698 {
4699         restarter_instance_state_t new_state = v->gv_state;
4700         int stn_transition, maint;
4701         int from, to;
4702         nvlist_t *attr;
4703         fmev_pri_t pri = FMEV_LOPRI;
4704         int raise = 0;
4705 
4706         if ((from = stn_restarter_state(old_state)) == -1 ||
4707             (to = stn_restarter_state(new_state)) == -1) {
4708                 stev_ct_bad_state++;
4709                 return;
4710         }
4711 
4712         stn_transition = from << 16 | to;
4713 
4714         maint = (to == SCF_STATE_MAINT || from == SCF_STATE_MAINT);
4715 
4716         if (maint) {
4717                 /*
4718                  * All transitions to/from maintenance state must raise
4719                  * an event.
4720                  */
4721                 raise++;
4722                 pri = FMEV_HIPRI;
4723                 stev_ct_maint++;
4724         } else if (reason == restarter_str_ct_ev_hwerr) {
4725                 /*
4726                  * All transitions caused by hardware fault must raise
4727                  * an event
4728                  */
4729                 raise++;
4730                 pri = FMEV_HIPRI;
4731                 stev_ct_hwerr++;
4732         } else if (stn_transition & v->gv_stn_tset) {
4733                 /*
4734                  * Specifically enabled event.
4735                  */
4736                 raise++;
4737                 stev_ct_service++;
4738         } else if (from == SCF_STATE_UNINIT) {
4739                 /*
4740                  * Only raise these if specifically selected above.
4741                  */
4742                 stev_ct_from_uninit++;
4743         } else if (stn_transition & stn_global &&
4744             (IS_ENABLED(v) == 1 || to == SCF_STATE_DISABLED)) {
4745                 raise++;
4746                 stev_ct_global++;
4747         } else {
4748                 stev_ct_noprefs++;
4749         }
4750 
4751         if (info_events_all) {
4752                 stev_ct_ovr_prefs++;
4753                 raise++;
4754         }
4755         if (!raise)
4756                 return;
4757 
4758         if (nvlist_alloc(&attr, NV_UNIQUE_NAME, 0) != 0 ||
4759             nvlist_add_string(attr, "fmri", v->gv_name) != 0 ||
4760             nvlist_add_uint32(attr, "reason-version",
4761             restarter_str_version()) || nvlist_add_string(attr, "reason-short",
4762             restarter_get_str_short(reason)) != 0 ||
4763             nvlist_add_string(attr, "reason-long",
4764             restarter_get_str_long(reason)) != 0 ||
4765             nvlist_add_int32(attr, "transition", stn_transition) != 0) {
4766                 log_framework(LOG_WARNING,
4767                     "FMEV: %s could not create nvlist for transition "
4768                     "event: %s\n", v->gv_name, strerror(errno));
4769                 nvlist_free(attr);
4770                 return;
4771         }
4772 
4773         if (fmev_rspublish_nvl(FMEV_RULESET_SMF, "state-transition",
4774             instance_state_str[new_state], pri, attr) != FMEV_SUCCESS) {
4775                 log_framework(LOG_DEBUG,
4776                     "FMEV: %s failed to publish transition event: %s\n",
4777                     v->gv_name, fmev_strerror(fmev_errno));
4778                 nvlist_free(attr);
4779         }
4780 }
4781 
4782 /*
4783  * Find the vertex for inst_name.  If it doesn't exist, return ENOENT.
4784  * Otherwise set its state to state.  If the instance has entered a state
4785  * which requires automatic action, take it (Uninitialized: do
4786  * dgraph_refresh_instance() without the snapshot update.  Disabled: if the
4787  * instance should be enabled, send _ENABLE.  Offline: if the instance should
4788  * be disabled, send _DISABLE, and if its dependencies are satisfied, send
4789  * _START.  Online, Degraded: if the instance wasn't running, update its start
4790  * snapshot.  Maintenance: no action.)
4791  *
4792  * Also fails with ECONNABORTED, or EINVAL if state is invalid.
4793  */
4794 static int
4795 dgraph_set_instance_state(scf_handle_t *h, const char *inst_name,
4796     protocol_states_t *states)
4797 {
4798         graph_vertex_t *v;
4799         int err = 0;
4800         restarter_instance_state_t old_state;
4801         restarter_instance_state_t state = states->ps_state;
4802         restarter_error_t serr = states->ps_err;
4803 
4804         MUTEX_LOCK(&dgraph_lock);
4805 
4806         v = vertex_get_by_name(inst_name);
4807         if (v == NULL) {
4808                 MUTEX_UNLOCK(&dgraph_lock);
4809                 return (ENOENT);
4810         }
4811 
4812         assert(v->gv_type == GVT_INST);
4813 
4814         switch (state) {
4815         case RESTARTER_STATE_UNINIT:
4816         case RESTARTER_STATE_DISABLED:
4817         case RESTARTER_STATE_OFFLINE:
4818         case RESTARTER_STATE_ONLINE:
4819         case RESTARTER_STATE_DEGRADED:
4820         case RESTARTER_STATE_MAINT:
4821                 break;
4822 
4823         default:
4824                 MUTEX_UNLOCK(&dgraph_lock);
4825                 return (EINVAL);
4826         }
4827 
4828         log_framework(LOG_DEBUG, "Graph noting %s %s -> %s.\n", v->gv_name,
4829             instance_state_str[v->gv_state], instance_state_str[state]);
4830 
4831         old_state = v->gv_state;
4832         v->gv_state = state;
4833 
4834         v->gv_reason = states->ps_reason;
4835         err = gt_transition(h, v, serr, old_state);
4836         if (err == 0 && v->gv_state != old_state) {
4837                 dgraph_state_transition_notify(v, old_state, states->ps_reason);
4838         }
4839 
4840         MUTEX_UNLOCK(&dgraph_lock);
4841         return (err);
4842 }
4843 
4844 /*
4845  * Handle state changes during milestone shutdown.  See
4846  * dgraph_set_milestone().  If the repository connection is broken,
4847  * ECONNABORTED will be returned, though a _DISABLE command will be sent for
4848  * the vertex anyway.
4849  */
4850 int
4851 vertex_subgraph_dependencies_shutdown(scf_handle_t *h, graph_vertex_t *v,
4852     restarter_instance_state_t old_state)
4853 {
4854         int was_up, now_up;
4855         int ret = 0;
4856 
4857         assert(v->gv_type == GVT_INST);
4858 
4859         /* Don't care if we're not going to a milestone. */
4860         if (milestone == NULL)
4861                 return (0);
4862 
4863         /* Don't care if we already finished coming down. */
4864         if (non_subgraph_svcs == 0)
4865                 return (0);
4866 
4867         /* Don't care if the service is in the subgraph. */
4868         if (v->gv_flags & GV_INSUBGRAPH)
4869                 return (0);
4870 
4871         /*
4872          * Update non_subgraph_svcs.  It is the number of non-subgraph
4873          * services which are in online, degraded, or offline.
4874          */
4875 
4876         was_up = up_state(old_state);
4877         now_up = up_state(v->gv_state);
4878 
4879         if (!was_up && now_up) {
4880                 ++non_subgraph_svcs;
4881         } else if (was_up && !now_up) {
4882                 --non_subgraph_svcs;
4883 
4884                 if (non_subgraph_svcs == 0) {
4885                         if (halting != -1) {
4886                                 do_uadmin();
4887                         } else if (go_single_user_mode || go_to_level1) {
4888                                 (void) startd_thread_create(single_user_thread,
4889                                     NULL);
4890                         }
4891                         return (0);
4892                 }
4893         }
4894 
4895         /* If this service is a leaf, it should be disabled. */
4896         if ((v->gv_flags & GV_ENABLED) && is_nonsubgraph_leaf(v)) {
4897                 int r;
4898 
4899                 r = disable_service_temporarily(v, h);
4900                 switch (r) {
4901                 case 0:
4902                         break;
4903 
4904                 case ECONNABORTED:
4905                         ret = ECONNABORTED;
4906                         break;
4907 
4908                 default:
4909                         bad_error("disable_service_temporarily", r);
4910                 }
4911         }
4912 
4913         /*
4914          * If the service just came down, propagate the disable to the newly
4915          * exposed leaves.
4916          */
4917         if (was_up && !now_up)
4918                 graph_walk_dependencies(v, disable_nonsubgraph_leaves,
4919                     (void *)h);
4920 
4921         return (ret);
4922 }
4923 
4924 /*
4925  * Decide whether to start up an sulogin thread after a service is
4926  * finished changing state.  Only need to do the full can_come_up()
4927  * evaluation if an instance is changing state, we're not halfway through
4928  * loading the thread, and we aren't shutting down or going to the single
4929  * user milestone.
4930  */
4931 void
4932 graph_transition_sulogin(restarter_instance_state_t state,
4933     restarter_instance_state_t old_state)
4934 {
4935         assert(MUTEX_HELD(&dgraph_lock));
4936 
4937         if (state != old_state && st->st_load_complete &&
4938             !go_single_user_mode && !go_to_level1 &&
4939             halting == -1) {
4940                 if (!sulogin_thread_running && !can_come_up()) {
4941                         (void) startd_thread_create(sulogin_thread, NULL);
4942                         sulogin_thread_running = B_TRUE;
4943                 }
4944         }
4945 }
4946 
4947 /*
4948  * Propagate a start, stop event, or a satisfiability event.
4949  *
4950  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4951  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4952  * full dependent graph to check for newly satisfied nodes.  This is
4953  * necessary for cases when non-direct dependents may be effected but direct
4954  * dependents may not (e.g. for optional_all evaluations, see the
4955  * propagate_satbility() comments).
4956  *
4957  * PROPAGATE_SAT should be used whenever a non-running service moves into
4958  * a state which can satisfy optional dependencies, like disabled or
4959  * maintenance.
4960  */
4961 void
4962 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4963     restarter_error_t rerr)
4964 {
4965         if (type == PROPAGATE_STOP) {
4966                 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4967         } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4968                 graph_walk_dependents(v, propagate_start, NULL);
4969 
4970                 if (type == PROPAGATE_SAT)
4971                         propagate_satbility(v);
4972         } else {
4973 #ifndef NDEBUG
4974                 uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4975                     __LINE__, type);
4976 #endif
4977                 abort();
4978         }
4979 }
4980 
4981 /*
4982  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4983  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4984  * all property group dependencies, and the dependency on the restarter,
4985  * disposing of vertices as appropriate.  If other vertices depend on this
4986  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4987  * returns 0.
4988  */
4989 static int
4990 dgraph_remove_instance(const char *fmri, scf_handle_t *h)
4991 {
4992         graph_vertex_t *v;
4993         graph_edge_t *e;
4994         uu_list_t *old_deps;
4995         int err;
4996 
4997         log_framework(LOG_DEBUG, "Graph engine: Removing %s.\n", fmri);
4998 
4999         MUTEX_LOCK(&dgraph_lock);
5000 
5001         v = vertex_get_by_name(fmri);
5002         if (v == NULL) {
5003                 MUTEX_UNLOCK(&dgraph_lock);
5004                 return (0);
5005         }
5006 
5007         /* Send restarter delete event. */
5008         if (v->gv_flags & GV_CONFIGURED)
5009                 graph_unset_restarter(v);
5010 
5011         if (milestone > MILESTONE_NONE) {
5012                 /*
5013                  * Make a list of v's current dependencies so we can
5014                  * reevaluate their GV_INSUBGRAPH flags after the dependencies
5015                  * are removed.
5016                  */
5017                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
5018 
5019                 err = uu_list_walk(v->gv_dependencies,
5020                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
5021                 assert(err == 0);
5022         }
5023 
5024         delete_instance_dependencies(v, B_TRUE);
5025 
5026         /*
5027          * Deleting an instance can both satisfy and unsatisfy dependencies,
5028          * depending on their type.  First propagate the stop as a RERR_RESTART
5029          * event -- deletion isn't a fault, just a normal stop.  This gives
5030          * dependent services the chance to do a clean shutdown.  Then, mark
5031          * the service as unconfigured and propagate the start event for the
5032          * optional_all dependencies that might have become satisfied.
5033          */
5034         graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
5035 
5036         v->gv_flags &= ~GV_CONFIGURED;
5037         v->gv_flags &= ~GV_DEATHROW;
5038 
5039         graph_walk_dependents(v, propagate_start, NULL);
5040         propagate_satbility(v);
5041 
5042         /*
5043          * If there are no (non-service) dependents, the vertex can be
5044          * completely removed.
5045          */
5046         if (v != milestone && v->gv_refs == 0 &&
5047             uu_list_numnodes(v->gv_dependents) == 1)
5048                 remove_inst_vertex(v);
5049 
5050         if (milestone > MILESTONE_NONE) {
5051                 void *cookie = NULL;
5052 
5053                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5054                         v = e->ge_vertex;
5055 
5056                         if (vertex_unref(v) == VERTEX_INUSE)
5057                                 while (eval_subgraph(v, h) == ECONNABORTED)
5058                                         libscf_handle_rebind(h);
5059 
5060                         startd_free(e, sizeof (*e));
5061                 }
5062 
5063                 uu_list_destroy(old_deps);
5064         }
5065 
5066         MUTEX_UNLOCK(&dgraph_lock);
5067 
5068         return (0);
5069 }
5070 
5071 /*
5072  * Return the eventual (maybe current) milestone in the form of a
5073  * legacy runlevel.
5074  */
5075 static char
5076 target_milestone_as_runlevel()
5077 {
5078         assert(MUTEX_HELD(&dgraph_lock));
5079 
5080         if (milestone == NULL)
5081                 return ('3');
5082         else if (milestone == MILESTONE_NONE)
5083                 return ('0');
5084 
5085         if (strcmp(milestone->gv_name, multi_user_fmri) == 0)
5086                 return ('2');
5087         else if (strcmp(milestone->gv_name, single_user_fmri) == 0)
5088                 return ('S');
5089         else if (strcmp(milestone->gv_name, multi_user_svr_fmri) == 0)
5090                 return ('3');
5091 
5092 #ifndef NDEBUG
5093         (void) fprintf(stderr, "%s:%d: Unknown milestone name \"%s\".\n",
5094             __FILE__, __LINE__, milestone->gv_name);
5095 #endif
5096         abort();
5097         /* NOTREACHED */
5098 }
5099 
5100 static struct {
5101         char    rl;
5102         int     sig;
5103 } init_sigs[] = {
5104         { 'S', SIGBUS },
5105         { '0', SIGINT },
5106         { '1', SIGQUIT },
5107         { '2', SIGILL },
5108         { '3', SIGTRAP },
5109         { '4', SIGIOT },
5110         { '5', SIGEMT },
5111         { '6', SIGFPE },
5112         { 0, 0 }
5113 };
5114 
5115 static void
5116 signal_init(char rl)
5117 {
5118         pid_t init_pid;
5119         int i;
5120 
5121         assert(MUTEX_HELD(&dgraph_lock));
5122 
5123         if (zone_getattr(getzoneid(), ZONE_ATTR_INITPID, &init_pid,
5124             sizeof (init_pid)) != sizeof (init_pid)) {
5125                 log_error(LOG_NOTICE, "Could not get pid to signal init.\n");
5126                 return;
5127         }
5128 
5129         for (i = 0; init_sigs[i].rl != 0; ++i)
5130                 if (init_sigs[i].rl == rl)
5131                         break;
5132 
5133         if (init_sigs[i].rl != 0) {
5134                 if (kill(init_pid, init_sigs[i].sig) != 0) {
5135                         switch (errno) {
5136                         case EPERM:
5137                         case ESRCH:
5138                                 log_error(LOG_NOTICE, "Could not signal init: "
5139                                     "%s.\n", strerror(errno));
5140                                 break;
5141 
5142                         case EINVAL:
5143                         default:
5144                                 bad_error("kill", errno);
5145                         }
5146                 }
5147         }
5148 }
5149 
5150 /*
5151  * This is called when one of the major milestones changes state, or when
5152  * init is signalled and tells us it was told to change runlevel.  We wait
5153  * to reach the milestone because this allows /etc/inittab entries to retain
5154  * some boot ordering: historically, entries could place themselves before/after
5155  * the running of /sbin/rcX scripts but we can no longer make the
5156  * distinction because the /sbin/rcX scripts no longer exist as punctuation
5157  * marks in /etc/inittab.
5158  *
5159  * Also, we only trigger an update when we reach the eventual target
5160  * milestone: without this, an /etc/inittab entry marked only for
5161  * runlevel 2 would be executed for runlevel 3, which is not how
5162  * /etc/inittab entries work.
5163  *
5164  * If we're single user coming online, then we set utmpx to the target
5165  * runlevel so that legacy scripts can work as expected.
5166  */
5167 static void
5168 graph_runlevel_changed(char rl, int online)
5169 {
5170         char trl;
5171 
5172         assert(MUTEX_HELD(&dgraph_lock));
5173 
5174         trl = target_milestone_as_runlevel();
5175 
5176         if (online) {
5177                 if (rl == trl) {
5178                         current_runlevel = trl;
5179                         signal_init(trl);
5180                 } else if (rl == 'S') {
5181                         /*
5182                          * At boot, set the entry early for the benefit of the
5183                          * legacy init scripts.
5184                          */
5185                         utmpx_set_runlevel(trl, 'S', B_FALSE);
5186                 }
5187         } else {
5188                 if (rl == '3' && trl == '2') {
5189                         current_runlevel = trl;
5190                         signal_init(trl);
5191                 } else if (rl == '2' && trl == 'S') {
5192                         current_runlevel = trl;
5193                         signal_init(trl);
5194                 }
5195         }
5196 }
5197 
5198 /*
5199  * Move to a backwards-compatible runlevel by executing the appropriate
5200  * /etc/rc?.d/K* scripts and/or setting the milestone.
5201  *
5202  * Returns
5203  *   0 - success
5204  *   ECONNRESET - success, but handle was reset
5205  *   ECONNABORTED - repository connection broken
5206  *   ECANCELED - pg was deleted
5207  */
5208 static int
5209 dgraph_set_runlevel(scf_propertygroup_t *pg, scf_property_t *prop)
5210 {
5211         char rl;
5212         scf_handle_t *h;
5213         int r;
5214         const char *ms = NULL;  /* what to commit as options/milestone */
5215         boolean_t rebound = B_FALSE;
5216         int mark_rl = 0;
5217 
5218         const char * const stop = "stop";
5219 
5220         r = libscf_extract_runlevel(prop, &rl);
5221         switch (r) {
5222         case 0:
5223                 break;
5224 
5225         case ECONNABORTED:
5226         case ECANCELED:
5227                 return (r);
5228 
5229         case EINVAL:
5230         case ENOENT:
5231                 log_error(LOG_WARNING, "runlevel property is misconfigured; "
5232                     "ignoring.\n");
5233                 /* delete the bad property */
5234                 goto nolock_out;
5235 
5236         default:
5237                 bad_error("libscf_extract_runlevel", r);
5238         }
5239 
5240         switch (rl) {
5241         case 's':
5242                 rl = 'S';
5243                 /* FALLTHROUGH */
5244 
5245         case 'S':
5246         case '2':
5247         case '3':
5248                 /*
5249                  * These cases cause a milestone change, so
5250                  * graph_runlevel_changed() will eventually deal with
5251                  * signalling init.
5252                  */
5253                 break;
5254 
5255         case '0':
5256         case '1':
5257         case '4':
5258         case '5':
5259         case '6':
5260                 mark_rl = 1;
5261                 break;
5262 
5263         default:
5264                 log_framework(LOG_NOTICE, "Unknown runlevel '%c'.\n", rl);
5265                 ms = NULL;
5266                 goto nolock_out;
5267         }
5268 
5269         h = scf_pg_handle(pg);
5270 
5271         MUTEX_LOCK(&dgraph_lock);
5272 
5273         /*
5274          * Since this triggers no milestone changes, force it by hand.
5275          */
5276         if (current_runlevel == '4' && rl == '3')
5277                 mark_rl = 1;
5278 
5279         /*
5280          * 1. If we are here after an "init X":
5281          *
5282          * init X
5283          *      init/lscf_set_runlevel()
5284          *              process_pg_event()
5285          *              dgraph_set_runlevel()
5286          *
5287          * then we haven't passed through graph_runlevel_changed() yet,
5288          * therefore 'current_runlevel' has not changed for sure but 'rl' has.
5289          * In consequence, if 'rl' is lower than 'current_runlevel', we change
5290          * the system runlevel and execute the appropriate /etc/rc?.d/K* scripts
5291          * past this test.
5292          *
5293          * 2. On the other hand, if we are here after a "svcadm milestone":
5294          *
5295          * svcadm milestone X
5296          *      dgraph_set_milestone()
5297          *              handle_graph_update_event()
5298          *              dgraph_set_instance_state()
5299          *              graph_post_X_[online|offline]()
5300          *              graph_runlevel_changed()
5301          *              signal_init()
5302          *                      init/lscf_set_runlevel()
5303          *                              process_pg_event()
5304          *                              dgraph_set_runlevel()
5305          *
5306          * then we already passed through graph_runlevel_changed() (by the way
5307          * of dgraph_set_milestone()) and 'current_runlevel' may have changed
5308          * and already be equal to 'rl' so we are going to return immediately
5309          * from dgraph_set_runlevel() without changing the system runlevel and
5310          * without executing the /etc/rc?.d/K* scripts.
5311          */
5312         if (rl == current_runlevel) {
5313                 ms = NULL;
5314                 goto out;
5315         }
5316 
5317         log_framework(LOG_DEBUG, "Changing to runlevel '%c'.\n", rl);
5318 
5319         /*
5320          * Make sure stop rc scripts see the new settings via who -r.
5321          */
5322         utmpx_set_runlevel(rl, current_runlevel, B_TRUE);
5323 
5324         /*
5325          * Some run levels don't have a direct correspondence to any
5326          * milestones, so we have to signal init directly.
5327          */
5328         if (mark_rl) {
5329                 current_runlevel = rl;
5330                 signal_init(rl);
5331         }
5332 
5333         switch (rl) {
5334         case 'S':
5335                 uu_warn("The system is coming down for administration.  "
5336                     "Please wait.\n");
5337                 fork_rc_script(rl, stop, B_FALSE);
5338                 ms = single_user_fmri;
5339                 go_single_user_mode = B_TRUE;
5340                 break;
5341 
5342         case '0':
5343                 halting_time = time(NULL);
5344                 fork_rc_script(rl, stop, B_TRUE);
5345                 halting = AD_HALT;
5346                 goto uadmin;
5347 
5348         case '5':
5349                 halting_time = time(NULL);
5350                 fork_rc_script(rl, stop, B_TRUE);
5351                 halting = AD_POWEROFF;
5352                 goto uadmin;
5353 
5354         case '6':
5355                 halting_time = time(NULL);
5356                 fork_rc_script(rl, stop, B_TRUE);
5357                 if (scf_is_fastboot_default() && getzoneid() == GLOBAL_ZONEID)
5358                         halting = AD_FASTREBOOT;
5359                 else
5360                         halting = AD_BOOT;
5361 
5362 uadmin:
5363                 uu_warn("The system is coming down.  Please wait.\n");
5364                 ms = "none";
5365 
5366                 /*
5367                  * We can't wait until all services are offline since this
5368                  * thread is responsible for taking them offline.  Instead we
5369                  * set halting to the second argument for uadmin() and call
5370                  * do_uadmin() from dgraph_set_instance_state() when
5371                  * appropriate.
5372                  */
5373                 break;
5374 
5375         case '1':
5376                 if (current_runlevel != 'S') {
5377                         uu_warn("Changing to state 1.\n");
5378                         fork_rc_script(rl, stop, B_FALSE);
5379                 } else {
5380                         uu_warn("The system is coming up for administration.  "
5381                             "Please wait.\n");
5382                 }
5383                 ms = single_user_fmri;
5384                 go_to_level1 = B_TRUE;
5385                 break;
5386 
5387         case '2':
5388                 if (current_runlevel == '3' || current_runlevel == '4')
5389                         fork_rc_script(rl, stop, B_FALSE);
5390                 ms = multi_user_fmri;
5391                 break;
5392 
5393         case '3':
5394         case '4':
5395                 ms = "all";
5396                 break;
5397 
5398         default:
5399 #ifndef NDEBUG
5400                 (void) fprintf(stderr, "%s:%d: Uncaught case %d ('%c').\n",
5401                     __FILE__, __LINE__, rl, rl);
5402 #endif
5403                 abort();
5404         }
5405 
5406 out:
5407         MUTEX_UNLOCK(&dgraph_lock);
5408 
5409 nolock_out:
5410         switch (r = libscf_clear_runlevel(pg, ms)) {
5411         case 0:
5412                 break;
5413 
5414         case ECONNABORTED:
5415                 libscf_handle_rebind(h);
5416                 rebound = B_TRUE;
5417                 goto nolock_out;
5418 
5419         case ECANCELED:
5420                 break;
5421 
5422         case EPERM:
5423         case EACCES:
5424         case EROFS:
5425                 log_error(LOG_NOTICE, "Could not delete \"%s/%s\" property: "
5426                     "%s.\n", SCF_PG_OPTIONS, "runlevel", strerror(r));
5427                 break;
5428 
5429         default:
5430                 bad_error("libscf_clear_runlevel", r);
5431         }
5432 
5433         return (rebound ? ECONNRESET : 0);
5434 }
5435 
5436 /*
5437  * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5438  * to the instances that are supposed to go offline during an
5439  * administrative disable operation.
5440  */
5441 static int
5442 mark_subtree(graph_edge_t *e, void *arg)
5443 {
5444         graph_vertex_t *v;
5445         int r;
5446 
5447         v = e->ge_vertex;
5448 
5449         /* If it's already in the subgraph, skip. */
5450         if (v->gv_flags & GV_TOOFFLINE)
5451                 return (UU_WALK_NEXT);
5452 
5453         switch (v->gv_type) {
5454         case GVT_INST:
5455                 /* If the instance is already disabled, skip it. */
5456                 if (!(v->gv_flags & GV_ENABLED))
5457                         return (UU_WALK_NEXT);
5458 
5459                 v->gv_flags |= GV_TOOFFLINE;
5460                 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5461                 break;
5462         case GVT_GROUP:
5463                 /*
5464                  * Skip all excluded and optional_all dependencies and decide
5465                  * whether to offline the service based on restart_on attribute.
5466                  */
5467                 if (is_depgrp_bypassed(v))
5468                         return (UU_WALK_NEXT);
5469                 break;
5470         }
5471 
5472         r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5473             0);
5474         assert(r == 0);
5475         return (UU_WALK_NEXT);
5476 }
5477 
5478 static int
5479 mark_subgraph(graph_edge_t *e, void *arg)
5480 {
5481         graph_vertex_t *v;
5482         int r;
5483         int optional = (int)arg;
5484 
5485         v = e->ge_vertex;
5486 
5487         /* If it's already in the subgraph, skip. */
5488         if (v->gv_flags & GV_INSUBGRAPH)
5489                 return (UU_WALK_NEXT);
5490 
5491         /*
5492          * Keep track if walk has entered an optional dependency group
5493          */
5494         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_OPTIONAL_ALL) {
5495                 optional = 1;
5496         }
5497         /*
5498          * Quit if we are in an optional dependency group and the instance
5499          * is disabled
5500          */
5501         if (optional && (v->gv_type == GVT_INST) &&
5502             (!(v->gv_flags & GV_ENBLD_NOOVR)))
5503                 return (UU_WALK_NEXT);
5504 
5505         v->gv_flags |= GV_INSUBGRAPH;
5506 
5507         /* Skip all excluded dependencies. */
5508         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
5509                 return (UU_WALK_NEXT);
5510 
5511         r = uu_list_walk(v->gv_dependencies, (uu_walk_fn_t *)mark_subgraph,
5512             (void *)optional, 0);
5513         assert(r == 0);
5514         return (UU_WALK_NEXT);
5515 }
5516 
5517 /*
5518  * Bring down all services which are not dependencies of fmri.  The
5519  * dependencies of fmri (direct & indirect) will constitute the "subgraph",
5520  * and will have the GV_INSUBGRAPH flag set.  The rest must be brought down,
5521  * which means the state is "disabled", "maintenance", or "uninitialized".  We
5522  * could consider "offline" to be down, and refrain from sending start
5523  * commands for such services, but that's not strictly necessary, so we'll
5524  * decline to intrude on the state machine.  It would probably confuse users
5525  * anyway.
5526  *
5527  * The services should be brought down in reverse-dependency order, so we
5528  * can't do it all at once here.  We initiate by override-disabling the leaves
5529  * of the dependency tree -- those services which are up but have no
5530  * dependents which are up.  When they come down,
5531  * vertex_subgraph_dependencies_shutdown() will override-disable the newly
5532  * exposed leaves.  Perseverance will ensure completion.
5533  *
5534  * Sometimes we need to take action when the transition is complete, like
5535  * start sulogin or halt the system.  To tell when we're done, we initialize
5536  * non_subgraph_svcs here to be the number of services which need to come
5537  * down.  As each does, we decrement the counter.  When it hits zero, we take
5538  * the appropriate action.  See vertex_subgraph_dependencies_shutdown().
5539  *
5540  * In case we're coming up, we also remove any enable-overrides for the
5541  * services which are dependencies of fmri.
5542  *
5543  * If norepository is true, the function will not change the repository.
5544  *
5545  * The decision to change the system run level in accordance with the milestone
5546  * is taken in dgraph_set_runlevel().
5547  *
5548  * Returns
5549  *   0 - success
5550  *   ECONNRESET - success, but handle was rebound
5551  *   EINVAL - fmri is invalid (error is logged)
5552  *   EALREADY - the milestone is already set to fmri
5553  *   ENOENT - a configured vertex does not exist for fmri (an error is logged)
5554  */
5555 static int
5556 dgraph_set_milestone(const char *fmri, scf_handle_t *h, boolean_t norepository)
5557 {
5558         const char *cfmri, *fs;
5559         graph_vertex_t *nm, *v;
5560         int ret = 0, r;
5561         scf_instance_t *inst;
5562         boolean_t isall, isnone, rebound = B_FALSE;
5563 
5564         /* Validate fmri */
5565         isall = (strcmp(fmri, "all") == 0);
5566         isnone = (strcmp(fmri, "none") == 0);
5567 
5568         if (!isall && !isnone) {
5569                 if (fmri_canonify(fmri, (char **)&cfmri, B_FALSE) == EINVAL)
5570                         goto reject;
5571 
5572                 if (strcmp(cfmri, single_user_fmri) != 0 &&
5573                     strcmp(cfmri, multi_user_fmri) != 0 &&
5574                     strcmp(cfmri, multi_user_svr_fmri) != 0) {
5575                         startd_free((void *)cfmri, max_scf_fmri_size);
5576 reject:
5577                         log_framework(LOG_WARNING,
5578                             "Rejecting request for invalid milestone \"%s\".\n",
5579                             fmri);
5580                         return (EINVAL);
5581                 }
5582         }
5583 
5584         inst = safe_scf_instance_create(h);
5585 
5586         MUTEX_LOCK(&dgraph_lock);
5587 
5588         if (milestone == NULL) {
5589                 if (isall) {
5590                         log_framework(LOG_DEBUG,
5591                             "Milestone already set to all.\n");
5592                         ret = EALREADY;
5593                         goto out;
5594                 }
5595         } else if (milestone == MILESTONE_NONE) {
5596                 if (isnone) {
5597                         log_framework(LOG_DEBUG,
5598                             "Milestone already set to none.\n");
5599                         ret = EALREADY;
5600                         goto out;
5601                 }
5602         } else {
5603                 if (!isall && !isnone &&
5604                     strcmp(cfmri, milestone->gv_name) == 0) {
5605                         log_framework(LOG_DEBUG,
5606                             "Milestone already set to %s.\n", cfmri);
5607                         ret = EALREADY;
5608                         goto out;
5609                 }
5610         }
5611 
5612         if (!isall && !isnone) {
5613                 nm = vertex_get_by_name(cfmri);
5614                 if (nm == NULL || !(nm->gv_flags & GV_CONFIGURED)) {
5615                         log_framework(LOG_WARNING, "Cannot set milestone to %s "
5616                             "because no such service exists.\n", cfmri);
5617                         ret = ENOENT;
5618                         goto out;
5619                 }
5620         }
5621 
5622         log_framework(LOG_DEBUG, "Changing milestone to %s.\n", fmri);
5623 
5624         /*
5625          * Set milestone, removing the old one if this was the last reference.
5626          */
5627         if (milestone > MILESTONE_NONE)
5628                 (void) vertex_unref(milestone);
5629 
5630         if (isall)
5631                 milestone = NULL;
5632         else if (isnone)
5633                 milestone = MILESTONE_NONE;
5634         else {
5635                 milestone = nm;
5636                 /* milestone should count as a reference */
5637                 vertex_ref(milestone);
5638         }
5639 
5640         /* Clear all GV_INSUBGRAPH bits. */
5641         for (v = uu_list_first(dgraph); v != NULL; v = uu_list_next(dgraph, v))
5642                 v->gv_flags &= ~GV_INSUBGRAPH;
5643 
5644         if (!isall && !isnone) {
5645                 /* Set GV_INSUBGRAPH for milestone & descendents. */
5646                 milestone->gv_flags |= GV_INSUBGRAPH;
5647 
5648                 r = uu_list_walk(milestone->gv_dependencies,
5649                     (uu_walk_fn_t *)mark_subgraph, NULL, 0);
5650                 assert(r == 0);
5651         }
5652 
5653         /* Un-override services in the subgraph & override-disable the rest. */
5654         if (norepository)
5655                 goto out;
5656 
5657         non_subgraph_svcs = 0;
5658         for (v = uu_list_first(dgraph);
5659             v != NULL;
5660             v = uu_list_next(dgraph, v)) {
5661                 if (v->gv_type != GVT_INST ||
5662                     (v->gv_flags & GV_CONFIGURED) == 0)
5663                         continue;
5664 
5665 again:
5666                 r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
5667                     NULL, NULL, SCF_DECODE_FMRI_EXACT);
5668                 if (r != 0) {
5669                         switch (scf_error()) {
5670                         case SCF_ERROR_CONNECTION_BROKEN:
5671                         default:
5672                                 libscf_handle_rebind(h);
5673                                 rebound = B_TRUE;
5674                                 goto again;
5675 
5676                         case SCF_ERROR_NOT_FOUND:
5677                                 continue;
5678 
5679                         case SCF_ERROR_HANDLE_MISMATCH:
5680                         case SCF_ERROR_INVALID_ARGUMENT:
5681                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5682                         case SCF_ERROR_NOT_BOUND:
5683                                 bad_error("scf_handle_decode_fmri",
5684                                     scf_error());
5685                         }
5686                 }
5687 
5688                 if (isall || (v->gv_flags & GV_INSUBGRAPH)) {
5689                         r = libscf_delete_enable_ovr(inst);
5690                         fs = "libscf_delete_enable_ovr";
5691                 } else {
5692                         assert(isnone || (v->gv_flags & GV_INSUBGRAPH) == 0);
5693 
5694                         /*
5695                          * Services which are up need to come down before
5696                          * we're done, but we can only disable the leaves
5697                          * here.
5698                          */
5699 
5700                         if (up_state(v->gv_state))
5701                                 ++non_subgraph_svcs;
5702 
5703                         /* If it's already disabled, don't bother. */
5704                         if ((v->gv_flags & GV_ENABLED) == 0)
5705                                 continue;
5706 
5707                         if (!is_nonsubgraph_leaf(v))
5708                                 continue;
5709 
5710                         r = libscf_set_enable_ovr(inst, 0);
5711                         fs = "libscf_set_enable_ovr";
5712                 }
5713                 switch (r) {
5714                 case 0:
5715                 case ECANCELED:
5716                         break;
5717 
5718                 case ECONNABORTED:
5719                         libscf_handle_rebind(h);
5720                         rebound = B_TRUE;
5721                         goto again;
5722 
5723                 case EPERM:
5724                 case EROFS:
5725                         log_error(LOG_WARNING,
5726                             "Could not set %s/%s for %s: %s.\n",
5727                             SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
5728                             v->gv_name, strerror(r));
5729                         break;
5730 
5731                 default:
5732                         bad_error(fs, r);
5733                 }
5734         }
5735 
5736         if (halting != -1) {
5737                 if (non_subgraph_svcs > 1)
5738                         uu_warn("%d system services are now being stopped.\n",
5739                             non_subgraph_svcs);
5740                 else if (non_subgraph_svcs == 1)
5741                         uu_warn("One system service is now being stopped.\n");
5742                 else if (non_subgraph_svcs == 0)
5743                         do_uadmin();
5744         }
5745 
5746         ret = rebound ? ECONNRESET : 0;
5747 
5748 out:
5749         MUTEX_UNLOCK(&dgraph_lock);
5750         if (!isall && !isnone)
5751                 startd_free((void *)cfmri, max_scf_fmri_size);
5752         scf_instance_destroy(inst);
5753         return (ret);
5754 }
5755 
5756 
5757 /*
5758  * Returns 0, ECONNABORTED, or EINVAL.
5759  */
5760 static int
5761 handle_graph_update_event(scf_handle_t *h, graph_protocol_event_t *e)
5762 {
5763         int r;
5764 
5765         switch (e->gpe_type) {
5766         case GRAPH_UPDATE_RELOAD_GRAPH:
5767                 log_error(LOG_WARNING,
5768                     "graph_event: reload graph unimplemented\n");
5769                 break;
5770 
5771         case GRAPH_UPDATE_STATE_CHANGE: {
5772                 protocol_states_t *states = e->gpe_data;
5773 
5774                 switch (r = dgraph_set_instance_state(h, e->gpe_inst, states)) {
5775                 case 0:
5776                 case ENOENT:
5777                         break;
5778 
5779                 case ECONNABORTED:
5780                         return (ECONNABORTED);
5781 
5782                 case EINVAL:
5783                 default:
5784 #ifndef NDEBUG
5785                         (void) fprintf(stderr, "dgraph_set_instance_state() "
5786                             "failed with unexpected error %d at %s:%d.\n", r,
5787                             __FILE__, __LINE__);
5788 #endif
5789                         abort();
5790                 }
5791 
5792                 startd_free(states, sizeof (protocol_states_t));
5793                 break;
5794         }
5795 
5796         default:
5797                 log_error(LOG_WARNING,
5798                     "graph_event_loop received an unknown event: %d\n",
5799                     e->gpe_type);
5800                 break;
5801         }
5802 
5803         return (0);
5804 }
5805 
5806 /*
5807  * graph_event_thread()
5808  *    Wait for state changes from the restarters.
5809  */
5810 /*ARGSUSED*/
5811 void *
5812 graph_event_thread(void *unused)
5813 {
5814         scf_handle_t *h;
5815         int err;
5816 
5817         h = libscf_handle_create_bound_loop();
5818 
5819         /*CONSTCOND*/
5820         while (1) {
5821                 graph_protocol_event_t *e;
5822 
5823                 MUTEX_LOCK(&gu->gu_lock);
5824 
5825                 while (gu->gu_wakeup == 0)
5826                         (void) pthread_cond_wait(&gu->gu_cv, &gu->gu_lock);
5827 
5828                 gu->gu_wakeup = 0;
5829 
5830                 while ((e = graph_event_dequeue()) != NULL) {
5831                         MUTEX_LOCK(&e->gpe_lock);
5832                         MUTEX_UNLOCK(&gu->gu_lock);
5833 
5834                         while ((err = handle_graph_update_event(h, e)) ==
5835                             ECONNABORTED)
5836                                 libscf_handle_rebind(h);
5837 
5838                         if (err == 0)
5839                                 graph_event_release(e);
5840                         else
5841                                 graph_event_requeue(e);
5842 
5843                         MUTEX_LOCK(&gu->gu_lock);
5844                 }
5845 
5846                 MUTEX_UNLOCK(&gu->gu_lock);
5847         }
5848 
5849         /*
5850          * Unreachable for now -- there's currently no graceful cleanup
5851          * called on exit().
5852          */
5853         MUTEX_UNLOCK(&gu->gu_lock);
5854         scf_handle_destroy(h);
5855         return (NULL);
5856 }
5857 
5858 static void
5859 set_initial_milestone(scf_handle_t *h)
5860 {
5861         scf_instance_t *inst;
5862         char *fmri, *cfmri;
5863         size_t sz;
5864         int r;
5865 
5866         inst = safe_scf_instance_create(h);
5867         fmri = startd_alloc(max_scf_fmri_size);
5868 
5869         /*
5870          * If -m milestone= was specified, we want to set options_ovr/milestone
5871          * to it.  Otherwise we want to read what the milestone should be set
5872          * to.  Either way we need our inst.
5873          */
5874 get_self:
5875         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
5876             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5877                 switch (scf_error()) {
5878                 case SCF_ERROR_CONNECTION_BROKEN:
5879                         libscf_handle_rebind(h);
5880                         goto get_self;
5881 
5882                 case SCF_ERROR_NOT_FOUND:
5883                         if (st->st_subgraph != NULL &&
5884                             st->st_subgraph[0] != '\0') {
5885                                 sz = strlcpy(fmri, st->st_subgraph,
5886                                     max_scf_fmri_size);
5887                                 assert(sz < max_scf_fmri_size);
5888                         } else {
5889                                 fmri[0] = '\0';
5890                         }
5891                         break;
5892 
5893                 case SCF_ERROR_INVALID_ARGUMENT:
5894                 case SCF_ERROR_CONSTRAINT_VIOLATED:
5895                 case SCF_ERROR_HANDLE_MISMATCH:
5896                 default:
5897                         bad_error("scf_handle_decode_fmri", scf_error());
5898                 }
5899         } else {
5900                 if (st->st_subgraph != NULL && st->st_subgraph[0] != '\0') {
5901                         scf_propertygroup_t *pg;
5902 
5903                         pg = safe_scf_pg_create(h);
5904 
5905                         sz = strlcpy(fmri, st->st_subgraph, max_scf_fmri_size);
5906                         assert(sz < max_scf_fmri_size);
5907 
5908                         r = libscf_inst_get_or_add_pg(inst, SCF_PG_OPTIONS_OVR,
5909                             SCF_PG_OPTIONS_OVR_TYPE, SCF_PG_OPTIONS_OVR_FLAGS,
5910                             pg);
5911                         switch (r) {
5912                         case 0:
5913                                 break;
5914 
5915                         case ECONNABORTED:
5916                                 libscf_handle_rebind(h);
5917                                 goto get_self;
5918 
5919                         case EPERM:
5920                         case EACCES:
5921                         case EROFS:
5922                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5923                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5924                                     SCF_PROPERTY_MILESTONE, strerror(r));
5925                                 /* FALLTHROUGH */
5926 
5927                         case ECANCELED:
5928                                 sz = strlcpy(fmri, st->st_subgraph,
5929                                     max_scf_fmri_size);
5930                                 assert(sz < max_scf_fmri_size);
5931                                 break;
5932 
5933                         default:
5934                                 bad_error("libscf_inst_get_or_add_pg", r);
5935                         }
5936 
5937                         r = libscf_clear_runlevel(pg, fmri);
5938                         switch (r) {
5939                         case 0:
5940                                 break;
5941 
5942                         case ECONNABORTED:
5943                                 libscf_handle_rebind(h);
5944                                 goto get_self;
5945 
5946                         case EPERM:
5947                         case EACCES:
5948                         case EROFS:
5949                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5950                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5951                                     SCF_PROPERTY_MILESTONE, strerror(r));
5952                                 /* FALLTHROUGH */
5953 
5954                         case ECANCELED:
5955                                 sz = strlcpy(fmri, st->st_subgraph,
5956                                     max_scf_fmri_size);
5957                                 assert(sz < max_scf_fmri_size);
5958                                 break;
5959 
5960                         default:
5961                                 bad_error("libscf_clear_runlevel", r);
5962                         }
5963 
5964                         scf_pg_destroy(pg);
5965                 } else {
5966                         scf_property_t *prop;
5967                         scf_value_t *val;
5968 
5969                         prop = safe_scf_property_create(h);
5970                         val = safe_scf_value_create(h);
5971 
5972                         r = libscf_get_milestone(inst, prop, val, fmri,
5973                             max_scf_fmri_size);
5974                         switch (r) {
5975                         case 0:
5976                                 break;
5977 
5978                         case ECONNABORTED:
5979                                 libscf_handle_rebind(h);
5980                                 goto get_self;
5981 
5982                         case EINVAL:
5983                                 log_error(LOG_WARNING, "Milestone property is "
5984                                     "misconfigured.  Defaulting to \"all\".\n");
5985                                 /* FALLTHROUGH */
5986 
5987                         case ECANCELED:
5988                         case ENOENT:
5989                                 fmri[0] = '\0';
5990                                 break;
5991 
5992                         default:
5993                                 bad_error("libscf_get_milestone", r);
5994                         }
5995 
5996                         scf_value_destroy(val);
5997                         scf_property_destroy(prop);
5998                 }
5999         }
6000 
6001         if (fmri[0] == '\0' || strcmp(fmri, "all") == 0)
6002                 goto out;
6003 
6004         if (strcmp(fmri, "none") != 0) {
6005 retry:
6006                 if (scf_handle_decode_fmri(h, fmri, NULL, NULL, inst, NULL,
6007                     NULL, SCF_DECODE_FMRI_EXACT) != 0) {
6008                         switch (scf_error()) {
6009                         case SCF_ERROR_INVALID_ARGUMENT:
6010                                 log_error(LOG_WARNING,
6011                                     "Requested milestone \"%s\" is invalid.  "
6012                                     "Reverting to \"all\".\n", fmri);
6013                                 goto out;
6014 
6015                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6016                                 log_error(LOG_WARNING, "Requested milestone "
6017                                     "\"%s\" does not specify an instance.  "
6018                                     "Reverting to \"all\".\n", fmri);
6019                                 goto out;
6020 
6021                         case SCF_ERROR_CONNECTION_BROKEN:
6022                                 libscf_handle_rebind(h);
6023                                 goto retry;
6024 
6025                         case SCF_ERROR_NOT_FOUND:
6026                                 log_error(LOG_WARNING, "Requested milestone "
6027                                     "\"%s\" not in repository.  Reverting to "
6028                                     "\"all\".\n", fmri);
6029                                 goto out;
6030 
6031                         case SCF_ERROR_HANDLE_MISMATCH:
6032                         default:
6033                                 bad_error("scf_handle_decode_fmri",
6034                                     scf_error());
6035                         }
6036                 }
6037 
6038                 r = fmri_canonify(fmri, &cfmri, B_FALSE);
6039                 assert(r == 0);
6040 
6041                 r = dgraph_add_instance(cfmri, inst, B_TRUE);
6042                 startd_free(cfmri, max_scf_fmri_size);
6043                 switch (r) {
6044                 case 0:
6045                         break;
6046 
6047                 case ECONNABORTED:
6048                         goto retry;
6049 
6050                 case EINVAL:
6051                         log_error(LOG_WARNING,
6052                             "Requested milestone \"%s\" is invalid.  "
6053                             "Reverting to \"all\".\n", fmri);
6054                         goto out;
6055 
6056                 case ECANCELED:
6057                         log_error(LOG_WARNING,
6058                             "Requested milestone \"%s\" not "
6059                             "in repository.  Reverting to \"all\".\n",
6060                             fmri);
6061                         goto out;
6062 
6063                 case EEXIST:
6064                 default:
6065                         bad_error("dgraph_add_instance", r);
6066                 }
6067         }
6068 
6069         log_console(LOG_INFO, "Booting to milestone \"%s\".\n", fmri);
6070 
6071         r = dgraph_set_milestone(fmri, h, B_FALSE);
6072         switch (r) {
6073         case 0:
6074         case ECONNRESET:
6075         case EALREADY:
6076                 break;
6077 
6078         case EINVAL:
6079         case ENOENT:
6080         default:
6081                 bad_error("dgraph_set_milestone", r);
6082         }
6083 
6084 out:
6085         startd_free(fmri, max_scf_fmri_size);
6086         scf_instance_destroy(inst);
6087 }
6088 
6089 void
6090 set_restart_milestone(scf_handle_t *h)
6091 {
6092         scf_instance_t *inst;
6093         scf_property_t *prop;
6094         scf_value_t *val;
6095         char *fmri;
6096         int r;
6097 
6098         inst = safe_scf_instance_create(h);
6099 
6100 get_self:
6101         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL,
6102             inst, NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
6103                 switch (scf_error()) {
6104                 case SCF_ERROR_CONNECTION_BROKEN:
6105                         libscf_handle_rebind(h);
6106                         goto get_self;
6107 
6108                 case SCF_ERROR_NOT_FOUND:
6109                         break;
6110 
6111                 case SCF_ERROR_INVALID_ARGUMENT:
6112                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6113                 case SCF_ERROR_HANDLE_MISMATCH:
6114                 default:
6115                         bad_error("scf_handle_decode_fmri", scf_error());
6116                 }
6117 
6118                 scf_instance_destroy(inst);
6119                 return;
6120         }
6121 
6122         prop = safe_scf_property_create(h);
6123         val = safe_scf_value_create(h);
6124         fmri = startd_alloc(max_scf_fmri_size);
6125 
6126         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6127         switch (r) {
6128         case 0:
6129                 break;
6130 
6131         case ECONNABORTED:
6132                 libscf_handle_rebind(h);
6133                 goto get_self;
6134 
6135         case ECANCELED:
6136         case ENOENT:
6137         case EINVAL:
6138                 goto out;
6139 
6140         default:
6141                 bad_error("libscf_get_milestone", r);
6142         }
6143 
6144         r = dgraph_set_milestone(fmri, h, B_TRUE);
6145         switch (r) {
6146         case 0:
6147         case ECONNRESET:
6148         case EALREADY:
6149         case EINVAL:
6150         case ENOENT:
6151                 break;
6152 
6153         default:
6154                 bad_error("dgraph_set_milestone", r);
6155         }
6156 
6157 out:
6158         startd_free(fmri, max_scf_fmri_size);
6159         scf_value_destroy(val);
6160         scf_property_destroy(prop);
6161         scf_instance_destroy(inst);
6162 }
6163 
6164 /*
6165  * void *graph_thread(void *)
6166  *
6167  * Graph management thread.
6168  */
6169 /*ARGSUSED*/
6170 void *
6171 graph_thread(void *arg)
6172 {
6173         scf_handle_t *h;
6174         int err;
6175 
6176         h = libscf_handle_create_bound_loop();
6177 
6178         if (st->st_initial)
6179                 set_initial_milestone(h);
6180 
6181         MUTEX_LOCK(&dgraph_lock);
6182         initial_milestone_set = B_TRUE;
6183         err = pthread_cond_broadcast(&initial_milestone_cv);
6184         assert(err == 0);
6185         MUTEX_UNLOCK(&dgraph_lock);
6186 
6187         libscf_populate_graph(h);
6188 
6189         if (!st->st_initial)
6190                 set_restart_milestone(h);
6191 
6192         MUTEX_LOCK(&st->st_load_lock);
6193         st->st_load_complete = 1;
6194         (void) pthread_cond_broadcast(&st->st_load_cv);
6195         MUTEX_UNLOCK(&st->st_load_lock);
6196 
6197         MUTEX_LOCK(&dgraph_lock);
6198         /*
6199          * Now that we've set st_load_complete we need to check can_come_up()
6200          * since if we booted to a milestone, then there won't be any more
6201          * state updates.
6202          */
6203         if (!go_single_user_mode && !go_to_level1 &&
6204             halting == -1) {
6205                 if (!sulogin_thread_running && !can_come_up()) {
6206                         (void) startd_thread_create(sulogin_thread, NULL);
6207                         sulogin_thread_running = B_TRUE;
6208                 }
6209         }
6210         MUTEX_UNLOCK(&dgraph_lock);
6211 
6212         (void) pthread_mutex_lock(&gu->gu_freeze_lock);
6213 
6214         /*CONSTCOND*/
6215         while (1) {
6216                 (void) pthread_cond_wait(&gu->gu_freeze_cv,
6217                     &gu->gu_freeze_lock);
6218         }
6219 
6220         /*
6221          * Unreachable for now -- there's currently no graceful cleanup
6222          * called on exit().
6223          */
6224         (void) pthread_mutex_unlock(&gu->gu_freeze_lock);
6225         scf_handle_destroy(h);
6226 
6227         return (NULL);
6228 }
6229 
6230 
6231 /*
6232  * int next_action()
6233  *   Given an array of timestamps 'a' with 'num' elements, find the
6234  *   lowest non-zero timestamp and return its index. If there are no
6235  *   non-zero elements, return -1.
6236  */
6237 static int
6238 next_action(hrtime_t *a, int num)
6239 {
6240         hrtime_t t = 0;
6241         int i = 0, smallest = -1;
6242 
6243         for (i = 0; i < num; i++) {
6244                 if (t == 0) {
6245                         t = a[i];
6246                         smallest = i;
6247                 } else if (a[i] != 0 && a[i] < t) {
6248                         t = a[i];
6249                         smallest = i;
6250                 }
6251         }
6252 
6253         if (t == 0)
6254                 return (-1);
6255         else
6256                 return (smallest);
6257 }
6258 
6259 /*
6260  * void process_actions()
6261  *   Process actions requested by the administrator. Possibilities include:
6262  *   refresh, restart, maintenance mode off, maintenance mode on,
6263  *   maintenance mode immediate, and degraded.
6264  *
6265  *   The set of pending actions is represented in the repository as a
6266  *   per-instance property group, with each action being a single property
6267  *   in that group.  This property group is converted to an array, with each
6268  *   action type having an array slot.  The actions in the array at the
6269  *   time process_actions() is called are acted on in the order of the
6270  *   timestamp (which is the value stored in the slot).  A value of zero
6271  *   indicates that there is no pending action of the type associated with
6272  *   a particular slot.
6273  *
6274  *   Sending an action event multiple times before the restarter has a
6275  *   chance to process that action will force it to be run at the last
6276  *   timestamp where it appears in the ordering.
6277  *
6278  *   Turning maintenance mode on trumps all other actions.
6279  *
6280  *   Returns 0 or ECONNABORTED.
6281  */
6282 static int
6283 process_actions(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst)
6284 {
6285         scf_property_t *prop = NULL;
6286         scf_value_t *val = NULL;
6287         scf_type_t type;
6288         graph_vertex_t *vertex;
6289         admin_action_t a;
6290         int i, ret = 0, r;
6291         hrtime_t action_ts[NACTIONS];
6292         char *inst_name;
6293 
6294         r = libscf_instance_get_fmri(inst, &inst_name);
6295         switch (r) {
6296         case 0:
6297                 break;
6298 
6299         case ECONNABORTED:
6300                 return (ECONNABORTED);
6301 
6302         case ECANCELED:
6303                 return (0);
6304 
6305         default:
6306                 bad_error("libscf_instance_get_fmri", r);
6307         }
6308 
6309         MUTEX_LOCK(&dgraph_lock);
6310 
6311         vertex = vertex_get_by_name(inst_name);
6312         if (vertex == NULL) {
6313                 MUTEX_UNLOCK(&dgraph_lock);
6314                 log_framework(LOG_DEBUG, "%s: Can't find graph vertex. "
6315                     "The instance must have been removed.\n", inst_name);
6316                 startd_free(inst_name, max_scf_fmri_size);
6317                 return (0);
6318         }
6319 
6320         prop = safe_scf_property_create(h);
6321         val = safe_scf_value_create(h);
6322 
6323         for (i = 0; i < NACTIONS; i++) {
6324                 if (scf_pg_get_property(pg, admin_actions[i], prop) != 0) {
6325                         switch (scf_error()) {
6326                         case SCF_ERROR_CONNECTION_BROKEN:
6327                         default:
6328                                 ret = ECONNABORTED;
6329                                 goto out;
6330 
6331                         case SCF_ERROR_DELETED:
6332                                 goto out;
6333 
6334                         case SCF_ERROR_NOT_FOUND:
6335                                 action_ts[i] = 0;
6336                                 continue;
6337 
6338                         case SCF_ERROR_HANDLE_MISMATCH:
6339                         case SCF_ERROR_INVALID_ARGUMENT:
6340                         case SCF_ERROR_NOT_SET:
6341                                 bad_error("scf_pg_get_property", scf_error());
6342                         }
6343                 }
6344 
6345                 if (scf_property_type(prop, &type) != 0) {
6346                         switch (scf_error()) {
6347                         case SCF_ERROR_CONNECTION_BROKEN:
6348                         default:
6349                                 ret = ECONNABORTED;
6350                                 goto out;
6351 
6352                         case SCF_ERROR_DELETED:
6353                                 action_ts[i] = 0;
6354                                 continue;
6355 
6356                         case SCF_ERROR_NOT_SET:
6357                                 bad_error("scf_property_type", scf_error());
6358                         }
6359                 }
6360 
6361                 if (type != SCF_TYPE_INTEGER) {
6362                         action_ts[i] = 0;
6363                         continue;
6364                 }
6365 
6366                 if (scf_property_get_value(prop, val) != 0) {
6367                         switch (scf_error()) {
6368                         case SCF_ERROR_CONNECTION_BROKEN:
6369                         default:
6370                                 ret = ECONNABORTED;
6371                                 goto out;
6372 
6373                         case SCF_ERROR_DELETED:
6374                                 goto out;
6375 
6376                         case SCF_ERROR_NOT_FOUND:
6377                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6378                                 action_ts[i] = 0;
6379                                 continue;
6380 
6381                         case SCF_ERROR_NOT_SET:
6382                         case SCF_ERROR_PERMISSION_DENIED:
6383                                 bad_error("scf_property_get_value",
6384                                     scf_error());
6385                         }
6386                 }
6387 
6388                 r = scf_value_get_integer(val, &action_ts[i]);
6389                 assert(r == 0);
6390         }
6391 
6392         a = ADMIN_EVENT_MAINT_ON_IMMEDIATE;
6393         if (action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ||
6394             action_ts[ADMIN_EVENT_MAINT_ON]) {
6395                 a = action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ?
6396                     ADMIN_EVENT_MAINT_ON_IMMEDIATE : ADMIN_EVENT_MAINT_ON;
6397 
6398                 vertex_send_event(vertex, admin_events[a]);
6399                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6400                 switch (r) {
6401                 case 0:
6402                 case EACCES:
6403                         break;
6404 
6405                 case ECONNABORTED:
6406                         ret = ECONNABORTED;
6407                         goto out;
6408 
6409                 case EPERM:
6410                         uu_die("Insufficient privilege.\n");
6411                         /* NOTREACHED */
6412 
6413                 default:
6414                         bad_error("libscf_unset_action", r);
6415                 }
6416         }
6417 
6418         while ((a = next_action(action_ts, NACTIONS)) != -1) {
6419                 log_framework(LOG_DEBUG,
6420                     "Graph: processing %s action for %s.\n", admin_actions[a],
6421                     inst_name);
6422 
6423                 if (a == ADMIN_EVENT_REFRESH) {
6424                         r = dgraph_refresh_instance(vertex, inst);
6425                         switch (r) {
6426                         case 0:
6427                         case ECANCELED:
6428                         case EINVAL:
6429                         case -1:
6430                                 break;
6431 
6432                         case ECONNABORTED:
6433                                 /* pg & inst are reset now, so just return. */
6434                                 ret = ECONNABORTED;
6435                                 goto out;
6436 
6437                         default:
6438                                 bad_error("dgraph_refresh_instance", r);
6439                         }
6440                 }
6441 
6442                 vertex_send_event(vertex, admin_events[a]);
6443 
6444                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6445                 switch (r) {
6446                 case 0:
6447                 case EACCES:
6448                         break;
6449 
6450                 case ECONNABORTED:
6451                         ret = ECONNABORTED;
6452                         goto out;
6453 
6454                 case EPERM:
6455                         uu_die("Insufficient privilege.\n");
6456                         /* NOTREACHED */
6457 
6458                 default:
6459                         bad_error("libscf_unset_action", r);
6460                 }
6461 
6462                 action_ts[a] = 0;
6463         }
6464 
6465 out:
6466         MUTEX_UNLOCK(&dgraph_lock);
6467 
6468         scf_property_destroy(prop);
6469         scf_value_destroy(val);
6470         startd_free(inst_name, max_scf_fmri_size);
6471         return (ret);
6472 }
6473 
6474 /*
6475  * inst and pg_name are scratch space, and are unset on entry.
6476  * Returns
6477  *   0 - success
6478  *   ECONNRESET - success, but repository handle rebound
6479  *   ECONNABORTED - repository connection broken
6480  */
6481 static int
6482 process_pg_event(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst,
6483     char *pg_name)
6484 {
6485         int r;
6486         scf_property_t *prop;
6487         scf_value_t *val;
6488         char *fmri;
6489         boolean_t rebound = B_FALSE, rebind_inst = B_FALSE;
6490 
6491         if (scf_pg_get_name(pg, pg_name, max_scf_value_size) < 0) {
6492                 switch (scf_error()) {
6493                 case SCF_ERROR_CONNECTION_BROKEN:
6494                 default:
6495                         return (ECONNABORTED);
6496 
6497                 case SCF_ERROR_DELETED:
6498                         return (0);
6499 
6500                 case SCF_ERROR_NOT_SET:
6501                         bad_error("scf_pg_get_name", scf_error());
6502                 }
6503         }
6504 
6505         if (strcmp(pg_name, SCF_PG_GENERAL) == 0 ||
6506             strcmp(pg_name, SCF_PG_GENERAL_OVR) == 0) {
6507                 r = dgraph_update_general(pg);
6508                 switch (r) {
6509                 case 0:
6510                 case ENOTSUP:
6511                 case ECANCELED:
6512                         return (0);
6513 
6514                 case ECONNABORTED:
6515                         return (ECONNABORTED);
6516 
6517                 case -1:
6518                         /* Error should have been logged. */
6519                         return (0);
6520 
6521                 default:
6522                         bad_error("dgraph_update_general", r);
6523                 }
6524         } else if (strcmp(pg_name, SCF_PG_RESTARTER_ACTIONS) == 0) {
6525                 if (scf_pg_get_parent_instance(pg, inst) != 0) {
6526                         switch (scf_error()) {
6527                         case SCF_ERROR_CONNECTION_BROKEN:
6528                                 return (ECONNABORTED);
6529 
6530                         case SCF_ERROR_DELETED:
6531                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6532                                 /* Ignore commands on services. */
6533                                 return (0);
6534 
6535                         case SCF_ERROR_NOT_BOUND:
6536                         case SCF_ERROR_HANDLE_MISMATCH:
6537                         case SCF_ERROR_NOT_SET:
6538                         default:
6539                                 bad_error("scf_pg_get_parent_instance",
6540                                     scf_error());
6541                         }
6542                 }
6543 
6544                 return (process_actions(h, pg, inst));
6545         }
6546 
6547         if (strcmp(pg_name, SCF_PG_OPTIONS) != 0 &&
6548             strcmp(pg_name, SCF_PG_OPTIONS_OVR) != 0)
6549                 return (0);
6550 
6551         /*
6552          * We only care about the options[_ovr] property groups of our own
6553          * instance, so get the fmri and compare.  Plus, once we know it's
6554          * correct, if the repository connection is broken we know exactly what
6555          * property group we were operating on, and can look it up again.
6556          */
6557         if (scf_pg_get_parent_instance(pg, inst) != 0) {
6558                 switch (scf_error()) {
6559                 case SCF_ERROR_CONNECTION_BROKEN:
6560                         return (ECONNABORTED);
6561 
6562                 case SCF_ERROR_DELETED:
6563                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6564                         return (0);
6565 
6566                 case SCF_ERROR_HANDLE_MISMATCH:
6567                 case SCF_ERROR_NOT_BOUND:
6568                 case SCF_ERROR_NOT_SET:
6569                 default:
6570                         bad_error("scf_pg_get_parent_instance",
6571                             scf_error());
6572                 }
6573         }
6574 
6575         switch (r = libscf_instance_get_fmri(inst, &fmri)) {
6576         case 0:
6577                 break;
6578 
6579         case ECONNABORTED:
6580                 return (ECONNABORTED);
6581 
6582         case ECANCELED:
6583                 return (0);
6584 
6585         default:
6586                 bad_error("libscf_instance_get_fmri", r);
6587         }
6588 
6589         if (strcmp(fmri, SCF_SERVICE_STARTD) != 0) {
6590                 startd_free(fmri, max_scf_fmri_size);
6591                 return (0);
6592         }
6593 
6594         /*
6595          * update the information events flag
6596          */
6597         if (strcmp(pg_name, SCF_PG_OPTIONS) == 0)
6598                 info_events_all = libscf_get_info_events_all(pg);
6599 
6600         prop = safe_scf_property_create(h);
6601         val = safe_scf_value_create(h);
6602 
6603         if (strcmp(pg_name, SCF_PG_OPTIONS_OVR) == 0) {
6604                 /* See if we need to set the runlevel. */
6605                 /* CONSTCOND */
6606                 if (0) {
6607 rebind_pg:
6608                         libscf_handle_rebind(h);
6609                         rebound = B_TRUE;
6610 
6611                         r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6612                         switch (r) {
6613                         case 0:
6614                                 break;
6615 
6616                         case ECONNABORTED:
6617                                 goto rebind_pg;
6618 
6619                         case ENOENT:
6620                                 goto out;
6621 
6622                         case EINVAL:
6623                         case ENOTSUP:
6624                                 bad_error("libscf_lookup_instance", r);
6625                         }
6626 
6627                         if (scf_instance_get_pg(inst, pg_name, pg) != 0) {
6628                                 switch (scf_error()) {
6629                                 case SCF_ERROR_DELETED:
6630                                 case SCF_ERROR_NOT_FOUND:
6631                                         goto out;
6632 
6633                                 case SCF_ERROR_CONNECTION_BROKEN:
6634                                         goto rebind_pg;
6635 
6636                                 case SCF_ERROR_HANDLE_MISMATCH:
6637                                 case SCF_ERROR_NOT_BOUND:
6638                                 case SCF_ERROR_NOT_SET:
6639                                 case SCF_ERROR_INVALID_ARGUMENT:
6640                                 default:
6641                                         bad_error("scf_instance_get_pg",
6642                                             scf_error());
6643                                 }
6644                         }
6645                 }
6646 
6647                 if (scf_pg_get_property(pg, "runlevel", prop) == 0) {
6648                         r = dgraph_set_runlevel(pg, prop);
6649                         switch (r) {
6650                         case ECONNRESET:
6651                                 rebound = B_TRUE;
6652                                 rebind_inst = B_TRUE;
6653                                 /* FALLTHROUGH */
6654 
6655                         case 0:
6656                                 break;
6657 
6658                         case ECONNABORTED:
6659                                 goto rebind_pg;
6660 
6661                         case ECANCELED:
6662                                 goto out;
6663 
6664                         default:
6665                                 bad_error("dgraph_set_runlevel", r);
6666                         }
6667                 } else {
6668                         switch (scf_error()) {
6669                         case SCF_ERROR_CONNECTION_BROKEN:
6670                         default:
6671                                 goto rebind_pg;
6672 
6673                         case SCF_ERROR_DELETED:
6674                                 goto out;
6675 
6676                         case SCF_ERROR_NOT_FOUND:
6677                                 break;
6678 
6679                         case SCF_ERROR_INVALID_ARGUMENT:
6680                         case SCF_ERROR_HANDLE_MISMATCH:
6681                         case SCF_ERROR_NOT_BOUND:
6682                         case SCF_ERROR_NOT_SET:
6683                                 bad_error("scf_pg_get_property", scf_error());
6684                         }
6685                 }
6686         }
6687 
6688         if (rebind_inst) {
6689 lookup_inst:
6690                 r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6691                 switch (r) {
6692                 case 0:
6693                         break;
6694 
6695                 case ECONNABORTED:
6696                         libscf_handle_rebind(h);
6697                         rebound = B_TRUE;
6698                         goto lookup_inst;
6699 
6700                 case ENOENT:
6701                         goto out;
6702 
6703                 case EINVAL:
6704                 case ENOTSUP:
6705                         bad_error("libscf_lookup_instance", r);
6706                 }
6707         }
6708 
6709         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6710         switch (r) {
6711         case 0:
6712                 break;
6713 
6714         case ECONNABORTED:
6715                 libscf_handle_rebind(h);
6716                 rebound = B_TRUE;
6717                 goto lookup_inst;
6718 
6719         case EINVAL:
6720                 log_error(LOG_NOTICE,
6721                     "%s/%s property of %s is misconfigured.\n", pg_name,
6722                     SCF_PROPERTY_MILESTONE, SCF_SERVICE_STARTD);
6723                 /* FALLTHROUGH */
6724 
6725         case ECANCELED:
6726         case ENOENT:
6727                 (void) strcpy(fmri, "all");
6728                 break;
6729 
6730         default:
6731                 bad_error("libscf_get_milestone", r);
6732         }
6733 
6734         r = dgraph_set_milestone(fmri, h, B_FALSE);
6735         switch (r) {
6736         case 0:
6737         case ECONNRESET:
6738         case EALREADY:
6739                 break;
6740 
6741         case EINVAL:
6742                 log_error(LOG_WARNING, "Milestone %s is invalid.\n", fmri);
6743                 break;
6744 
6745         case ENOENT:
6746                 log_error(LOG_WARNING, "Milestone %s does not exist.\n", fmri);
6747                 break;
6748 
6749         default:
6750                 bad_error("dgraph_set_milestone", r);
6751         }
6752 
6753 out:
6754         startd_free(fmri, max_scf_fmri_size);
6755         scf_value_destroy(val);
6756         scf_property_destroy(prop);
6757 
6758         return (rebound ? ECONNRESET : 0);
6759 }
6760 
6761 /*
6762  * process_delete() deletes an instance from the dgraph if 'fmri' is an
6763  * instance fmri or if 'fmri' matches the 'general' property group of an
6764  * instance (or the 'general/enabled' property).
6765  *
6766  * 'fmri' may be overwritten and cannot be trusted on return by the caller.
6767  */
6768 static void
6769 process_delete(char *fmri, scf_handle_t *h)
6770 {
6771         char *lfmri, *end_inst_fmri;
6772         const char *inst_name = NULL;
6773         const char *pg_name = NULL;
6774         const char *prop_name = NULL;
6775 
6776         lfmri = safe_strdup(fmri);
6777 
6778         /* Determine if the FMRI is a property group or instance */
6779         if (scf_parse_svc_fmri(lfmri, NULL, NULL, &inst_name, &pg_name,
6780             &prop_name) != SCF_SUCCESS) {
6781                 log_error(LOG_WARNING,
6782                     "Received invalid FMRI \"%s\" from repository server.\n",
6783                     fmri);
6784         } else if (inst_name != NULL && pg_name == NULL) {
6785                 (void) dgraph_remove_instance(fmri, h);
6786         } else if (inst_name != NULL && pg_name != NULL) {
6787                 /*
6788                  * If we're deleting the 'general' property group or
6789                  * 'general/enabled' property then the whole instance
6790                  * must be removed from the dgraph.
6791                  */
6792                 if (strcmp(pg_name, SCF_PG_GENERAL) != 0) {
6793                         free(lfmri);
6794                         return;
6795                 }
6796 
6797                 if (prop_name != NULL &&
6798                     strcmp(prop_name, SCF_PROPERTY_ENABLED) != 0) {
6799                         free(lfmri);
6800                         return;
6801                 }
6802 
6803                 /*
6804                  * Because the instance has already been deleted from the
6805                  * repository, we cannot use any scf_ functions to retrieve
6806                  * the instance FMRI however we can easily reconstruct it
6807                  * manually.
6808                  */
6809                 end_inst_fmri = strstr(fmri, SCF_FMRI_PROPERTYGRP_PREFIX);
6810                 if (end_inst_fmri == NULL)
6811                         bad_error("process_delete", 0);
6812 
6813                 end_inst_fmri[0] = '\0';
6814 
6815                 (void) dgraph_remove_instance(fmri, h);
6816         }
6817 
6818         free(lfmri);
6819 }
6820 
6821 /*ARGSUSED*/
6822 void *
6823 repository_event_thread(void *unused)
6824 {
6825         scf_handle_t *h;
6826         scf_propertygroup_t *pg;
6827         scf_instance_t *inst;
6828         char *fmri = startd_alloc(max_scf_fmri_size);
6829         char *pg_name = startd_alloc(max_scf_value_size);
6830         int r;
6831 
6832         h = libscf_handle_create_bound_loop();
6833 
6834         pg = safe_scf_pg_create(h);
6835         inst = safe_scf_instance_create(h);
6836 
6837 retry:
6838         if (_scf_notify_add_pgtype(h, SCF_GROUP_FRAMEWORK) != SCF_SUCCESS) {
6839                 if (scf_error() == SCF_ERROR_CONNECTION_BROKEN) {
6840                         libscf_handle_rebind(h);
6841                 } else {
6842                         log_error(LOG_WARNING,
6843                             "Couldn't set up repository notification "
6844                             "for property group type %s: %s\n",
6845                             SCF_GROUP_FRAMEWORK, scf_strerror(scf_error()));
6846 
6847                         (void) sleep(1);
6848                 }
6849 
6850                 goto retry;
6851         }
6852 
6853         /*CONSTCOND*/
6854         while (1) {
6855                 ssize_t res;
6856 
6857                 /* Note: fmri is only set on delete events. */
6858                 res = _scf_notify_wait(pg, fmri, max_scf_fmri_size);
6859                 if (res < 0) {
6860                         libscf_handle_rebind(h);
6861                         goto retry;
6862                 } else if (res == 0) {
6863                         /*
6864                          * property group modified.  inst and pg_name are
6865                          * pre-allocated scratch space.
6866                          */
6867                         if (scf_pg_update(pg) < 0) {
6868                                 switch (scf_error()) {
6869                                 case SCF_ERROR_DELETED:
6870                                         continue;
6871 
6872                                 case SCF_ERROR_CONNECTION_BROKEN:
6873                                         log_error(LOG_WARNING,
6874                                             "Lost repository event due to "
6875                                             "disconnection.\n");
6876                                         libscf_handle_rebind(h);
6877                                         goto retry;
6878 
6879                                 case SCF_ERROR_NOT_BOUND:
6880                                 case SCF_ERROR_NOT_SET:
6881                                 default:
6882                                         bad_error("scf_pg_update", scf_error());
6883                                 }
6884                         }
6885 
6886                         r = process_pg_event(h, pg, inst, pg_name);
6887                         switch (r) {
6888                         case 0:
6889                                 break;
6890 
6891                         case ECONNABORTED:
6892                                 log_error(LOG_WARNING, "Lost repository event "
6893                                     "due to disconnection.\n");
6894                                 libscf_handle_rebind(h);
6895                                 /* FALLTHROUGH */
6896 
6897                         case ECONNRESET:
6898                                 goto retry;
6899 
6900                         default:
6901                                 bad_error("process_pg_event", r);
6902                         }
6903                 } else {
6904                         /*
6905                          * Service, instance, or pg deleted.
6906                          * Don't trust fmri on return.
6907                          */
6908                         process_delete(fmri, h);
6909                 }
6910         }
6911 
6912         /*NOTREACHED*/
6913         return (NULL);
6914 }
6915 
6916 void
6917 graph_engine_start()
6918 {
6919         int err;
6920 
6921         (void) startd_thread_create(graph_thread, NULL);
6922 
6923         MUTEX_LOCK(&dgraph_lock);
6924         while (!initial_milestone_set) {
6925                 err = pthread_cond_wait(&initial_milestone_cv, &dgraph_lock);
6926                 assert(err == 0);
6927         }
6928         MUTEX_UNLOCK(&dgraph_lock);
6929 
6930         (void) startd_thread_create(repository_event_thread, NULL);
6931         (void) startd_thread_create(graph_event_thread, NULL);
6932 }