1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
  25  * Copyright 2016 Toomas Soome <tsoome@me.com>
  26  */
  27 
  28 /*
  29  * graph.c - master restarter graph engine
  30  *
  31  *   The graph engine keeps a dependency graph of all service instances on the
  32  *   system, as recorded in the repository.  It decides when services should
  33  *   be brought up or down based on service states and dependencies and sends
  34  *   commands to restarters to effect any changes.  It also executes
  35  *   administrator commands sent by svcadm via the repository.
  36  *
  37  *   The graph is stored in uu_list_t *dgraph and its vertices are
  38  *   graph_vertex_t's, each of which has a name and an integer id unique to
  39  *   its name (see dict.c).  A vertex's type attribute designates the type
  40  *   of object it represents: GVT_INST for service instances, GVT_SVC for
  41  *   service objects (since service instances may depend on another service,
  42  *   rather than service instance), GVT_FILE for files (which services may
  43  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
  44  *   vertices are necessary because dependency lists may have particular
  45  *   grouping types (require any, require all, optional, or exclude) and
  46  *   event-propagation characteristics.
  47  *
  48  *   The initial graph is built by libscf_populate_graph() invoking
  49  *   dgraph_add_instance() for each instance in the repository.  The function
  50  *   adds a GVT_SVC vertex for the service if one does not already exist, adds
  51  *   a GVT_INST vertex named by the FMRI of the instance, and sets up the edges.
  52  *   The resulting web of vertices & edges associated with an instance's vertex
  53  *   includes
  54  *
  55  *     - an edge from the GVT_SVC vertex for the instance's service
  56  *
  57  *     - an edge to the GVT_INST vertex of the instance's resarter, if its
  58  *       restarter is not svc.startd
  59  *
  60  *     - edges from other GVT_INST vertices if the instance is a restarter
  61  *
  62  *     - for each dependency property group in the instance's "running"
  63  *       snapshot, an edge to a GVT_GROUP vertex named by the FMRI of the
  64  *       instance and the name of the property group
  65  *
  66  *     - for each value of the "entities" property in each dependency property
  67  *       group, an edge from the corresponding GVT_GROUP vertex to a
  68  *       GVT_INST, GVT_SVC, or GVT_FILE vertex
  69  *
  70  *     - edges from GVT_GROUP vertices for each dependent instance
  71  *
  72  *   After the edges are set up the vertex's GV_CONFIGURED flag is set.  If
  73  *   there are problems, or if a service is mentioned in a dependency but does
  74  *   not exist in the repository, the GV_CONFIGURED flag will be clear.
  75  *
  76  *   The graph and all of its vertices are protected by the dgraph_lock mutex.
  77  *   See restarter.c for more information.
  78  *
  79  *   The properties of an instance fall into two classes: immediate and
  80  *   snapshotted.  Immediate properties should have an immediate effect when
  81  *   changed.  Snapshotted properties should be read from a snapshot, so they
  82  *   only change when the snapshot changes.  The immediate properties used by
  83  *   the graph engine are general/enabled, general/restarter, and the properties
  84  *   in the restarter_actions property group.  Since they are immediate, they
  85  *   are not read out of a snapshot.  The snapshotted properties used by the
  86  *   graph engine are those in the property groups with type "dependency" and
  87  *   are read out of the "running" snapshot.  The "running" snapshot is created
  88  *   by the the graph engine as soon as possible, and it is updated, along with
  89  *   in-core copies of the data (dependency information for the graph engine) on
  90  *   receipt of the refresh command from svcadm.  In addition, the graph engine
  91  *   updates the "start" snapshot from the "running" snapshot whenever a service
  92  *   comes online.
  93  *
  94  *   When a DISABLE event is requested by the administrator, svc.startd shutdown
  95  *   the dependents first before shutting down the requested service.
  96  *   In graph_enable_by_vertex, we create a subtree that contains the dependent
  97  *   vertices by marking those vertices with the GV_TOOFFLINE flag. And we mark
  98  *   the vertex to disable with the GV_TODISABLE flag. Once the tree is created,
  99  *   we send the _ADMIN_DISABLE event to the leaves. The leaves will then
 100  *   transition from STATE_ONLINE/STATE_DEGRADED to STATE_OFFLINE/STATE_MAINT.
 101  *   In gt_enter_offline and gt_enter_maint if the vertex was in a subtree then
 102  *   we clear the GV_TOOFFLINE flag and walk the dependencies to offline the new
 103  *   exposed leaves. We do the same until we reach the last leaf (the one with
 104  *   the GV_TODISABLE flag). If the vertex to disable is also part of a larger
 105  *   subtree (eg. multiple DISABLE events on vertices in the same subtree) then
 106  *   once the first vertex is disabled (GV_TODISABLE flag is removed), we
 107  *   continue to propagate the offline event to the vertex's dependencies.
 108  *
 109  *
 110  * SMF state transition notifications
 111  *
 112  *   When an instance of a service managed by SMF changes state, svc.startd may
 113  *   publish a GPEC sysevent. All transitions to or from maintenance, a
 114  *   transition cause by a hardware error will generate an event.
 115  *   Other transitions will generate an event if there exist notification
 116  *   parameter for that transition. Notification parameters are stored in the
 117  *   SMF repository for the service/instance they refer to. System-wide
 118  *   notification parameters are stored in the global instance.
 119  *   svc.startd can be told to send events for all SMF state transitions despite
 120  *   of notification parameters by setting options/info_events_all to true in
 121  *   restarter:default
 122  *
 123  *   The set of transitions that generate events is cached in the
 124  *   dgraph_vertex_t gv_stn_tset for service/instance and in the global
 125  *   stn_global for the system-wide set. They are re-read when instances are
 126  *   refreshed.
 127  *
 128  *   The GPEC events published by svc.startd are consumed by fmd(1M). After
 129  *   processing these events, fmd(1M) publishes the processed events to
 130  *   notification agents. The notification agents read the notification
 131  *   parameters from the SMF repository through libscf(3LIB) interfaces and send
 132  *   the notification, or not, based on those parameters.
 133  *
 134  *   Subscription and publishing to the GPEC channels is done with the
 135  *   libfmevent(3LIB) wrappers fmev_[r]publish_*() and
 136  *   fmev_shdl_(un)subscribe().
 137  *
 138  */
 139 
 140 #include <sys/uadmin.h>
 141 #include <sys/wait.h>
 142 
 143 #include <assert.h>
 144 #include <errno.h>
 145 #include <fcntl.h>
 146 #include <fm/libfmevent.h>
 147 #include <libscf.h>
 148 #include <libscf_priv.h>
 149 #include <librestart.h>
 150 #include <libuutil.h>
 151 #include <locale.h>
 152 #include <poll.h>
 153 #include <pthread.h>
 154 #include <signal.h>
 155 #include <stddef.h>
 156 #include <stdio.h>
 157 #include <stdlib.h>
 158 #include <string.h>
 159 #include <strings.h>
 160 #include <sys/statvfs.h>
 161 #include <sys/uadmin.h>
 162 #include <zone.h>
 163 #if defined(__x86)
 164 #include <libbe.h>
 165 #endif  /* __x86 */
 166 
 167 #include "startd.h"
 168 #include "protocol.h"
 169 
 170 
 171 #define MILESTONE_NONE  ((graph_vertex_t *)1)
 172 
 173 #define CONSOLE_LOGIN_FMRI      "svc:/system/console-login:default"
 174 #define FS_MINIMAL_FMRI         "svc:/system/filesystem/minimal:default"
 175 
 176 #define VERTEX_REMOVED  0       /* vertex has been freed  */
 177 #define VERTEX_INUSE    1       /* vertex is still in use */
 178 
 179 #define IS_ENABLED(v) ((v)->gv_flags & (GV_ENABLED | GV_ENBLD_NOOVR))
 180 
 181 /*
 182  * stn_global holds the tset for the system wide notification parameters.
 183  * It is updated on refresh of svc:/system/svc/global:default
 184  *
 185  * There are two assumptions that relax the need for a mutex:
 186  *     1. 32-bit value assignments are atomic
 187  *     2. Its value is consumed only in one point at
 188  *     dgraph_state_transition_notify(). There are no test and set races.
 189  *
 190  *     If either assumption is broken, we'll need a mutex to synchronize
 191  *     access to stn_global
 192  */
 193 int32_t stn_global;
 194 /*
 195  * info_events_all holds a flag to override notification parameters and send
 196  * Information events for all state transitions.
 197  * same about the need of a mutex here.
 198  */
 199 int info_events_all;
 200 
 201 /*
 202  * Services in these states are not considered 'down' by the
 203  * milestone/shutdown code.
 204  */
 205 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
 206         (state) == RESTARTER_STATE_DEGRADED || \
 207         (state) == RESTARTER_STATE_OFFLINE)
 208 
 209 #define is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
 210         ((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \
 211         (v->gv_depgroup == DEPGRP_OPTIONAL_ALL) || \
 212         (v->gv_restart < RERR_RESTART)))
 213 
 214 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
 215 static uu_list_t *dgraph;
 216 static pthread_mutex_t dgraph_lock;
 217 
 218 /*
 219  * milestone indicates the current subgraph.  When NULL, it is the entire
 220  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
 221  * services on which the target vertex depends.
 222  */
 223 static graph_vertex_t *milestone = NULL;
 224 static boolean_t initial_milestone_set = B_FALSE;
 225 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
 226 
 227 /* protected by dgraph_lock */
 228 static boolean_t sulogin_thread_running = B_FALSE;
 229 static boolean_t sulogin_running = B_FALSE;
 230 static boolean_t console_login_ready = B_FALSE;
 231 
 232 /* Number of services to come down to complete milestone transition. */
 233 static uint_t non_subgraph_svcs;
 234 
 235 /*
 236  * These variables indicate what should be done when we reach the milestone
 237  * target milestone, i.e., when non_subgraph_svcs == 0.  They are acted upon in
 238  * dgraph_set_instance_state().
 239  */
 240 static int halting = -1;
 241 static boolean_t go_single_user_mode = B_FALSE;
 242 static boolean_t go_to_level1 = B_FALSE;
 243 
 244 /*
 245  * Tracks when we started halting.
 246  */
 247 static time_t halting_time = 0;
 248 
 249 /*
 250  * This tracks the legacy runlevel to ensure we signal init and manage
 251  * utmpx entries correctly.
 252  */
 253 static char current_runlevel = '\0';
 254 
 255 /* Number of single user threads currently running */
 256 static pthread_mutex_t single_user_thread_lock;
 257 static int single_user_thread_count = 0;
 258 
 259 /* Statistics for dependency cycle-checking */
 260 static u_longlong_t dep_inserts = 0;
 261 static u_longlong_t dep_cycle_ns = 0;
 262 static u_longlong_t dep_insert_ns = 0;
 263 
 264 
 265 static const char * const emsg_invalid_restarter =
 266         "Transitioning %s to maintenance, restarter FMRI %s is invalid "
 267         "(see 'svcs -xv' for details).\n";
 268 static const char * const console_login_fmri = CONSOLE_LOGIN_FMRI;
 269 static const char * const single_user_fmri = SCF_MILESTONE_SINGLE_USER;
 270 static const char * const multi_user_fmri = SCF_MILESTONE_MULTI_USER;
 271 static const char * const multi_user_svr_fmri = SCF_MILESTONE_MULTI_USER_SERVER;
 272 
 273 
 274 /*
 275  * These services define the system being "up".  If none of them can come
 276  * online, then we will run sulogin on the console.  Note that the install ones
 277  * are for the miniroot and when installing CDs after the first.  can_come_up()
 278  * does the decision making, and an sulogin_thread() runs sulogin, which can be
 279  * started by dgraph_set_instance_state() or single_user_thread().
 280  *
 281  * NOTE: can_come_up() relies on SCF_MILESTONE_SINGLE_USER being the first
 282  * entry, which is only used when booting_to_single_user (boot -s) is set.
 283  * This is because when doing a "boot -s", sulogin is started from specials.c
 284  * after milestone/single-user comes online, for backwards compatibility.
 285  * In this case, SCF_MILESTONE_SINGLE_USER needs to be part of up_svcs
 286  * to ensure sulogin will be spawned if milestone/single-user cannot be reached.
 287  */
 288 static const char * const up_svcs[] = {
 289         SCF_MILESTONE_SINGLE_USER,
 290         CONSOLE_LOGIN_FMRI,
 291         "svc:/system/install-setup:default",
 292         "svc:/system/install:default",
 293         NULL
 294 };
 295 
 296 /* This array must have an element for each non-NULL element of up_svcs[]. */
 297 static graph_vertex_t *up_svcs_p[] = { NULL, NULL, NULL, NULL };
 298 
 299 /* These are for seed repository magic.  See can_come_up(). */
 300 static const char * const manifest_import = SCF_INSTANCE_MI;
 301 static graph_vertex_t *manifest_import_p = NULL;
 302 
 303 
 304 static char target_milestone_as_runlevel(void);
 305 static void graph_runlevel_changed(char rl, int online);
 306 static int dgraph_set_milestone(const char *, scf_handle_t *, boolean_t);
 307 static boolean_t should_be_in_subgraph(graph_vertex_t *v);
 308 static int mark_subtree(graph_edge_t *, void *);
 309 static boolean_t insubtree_dependents_down(graph_vertex_t *);
 310 
 311 /*
 312  * graph_vertex_compare()
 313  *      This function can compare either int *id or * graph_vertex_t *gv
 314  *      values, as the vertex id is always the first element of a
 315  *      graph_vertex structure.
 316  */
 317 /* ARGSUSED */
 318 static int
 319 graph_vertex_compare(const void *lc_arg, const void *rc_arg, void *private)
 320 {
 321         int lc_id = ((const graph_vertex_t *)lc_arg)->gv_id;
 322         int rc_id = *(int *)rc_arg;
 323 
 324         if (lc_id > rc_id)
 325                 return (1);
 326         if (lc_id < rc_id)
 327                 return (-1);
 328         return (0);
 329 }
 330 
 331 void
 332 graph_init()
 333 {
 334         graph_edge_pool = startd_list_pool_create("graph_edges",
 335             sizeof (graph_edge_t), offsetof(graph_edge_t, ge_link), NULL,
 336             UU_LIST_POOL_DEBUG);
 337         assert(graph_edge_pool != NULL);
 338 
 339         graph_vertex_pool = startd_list_pool_create("graph_vertices",
 340             sizeof (graph_vertex_t), offsetof(graph_vertex_t, gv_link),
 341             graph_vertex_compare, UU_LIST_POOL_DEBUG);
 342         assert(graph_vertex_pool != NULL);
 343 
 344         (void) pthread_mutex_init(&dgraph_lock, &mutex_attrs);
 345         (void) pthread_mutex_init(&single_user_thread_lock, &mutex_attrs);
 346         dgraph = startd_list_create(graph_vertex_pool, NULL, UU_LIST_SORTED);
 347         assert(dgraph != NULL);
 348 
 349         if (!st->st_initial)
 350                 current_runlevel = utmpx_get_runlevel();
 351 
 352         log_framework(LOG_DEBUG, "Initialized graph\n");
 353 }
 354 
 355 static graph_vertex_t *
 356 vertex_get_by_name(const char *name)
 357 {
 358         int id;
 359 
 360         assert(MUTEX_HELD(&dgraph_lock));
 361 
 362         id = dict_lookup_byname(name);
 363         if (id == -1)
 364                 return (NULL);
 365 
 366         return (uu_list_find(dgraph, &id, NULL, NULL));
 367 }
 368 
 369 static graph_vertex_t *
 370 vertex_get_by_id(int id)
 371 {
 372         assert(MUTEX_HELD(&dgraph_lock));
 373 
 374         if (id == -1)
 375                 return (NULL);
 376 
 377         return (uu_list_find(dgraph, &id, NULL, NULL));
 378 }
 379 
 380 /*
 381  * Creates a new vertex with the given name, adds it to the graph, and returns
 382  * a pointer to it.  The graph lock must be held by this thread on entry.
 383  */
 384 static graph_vertex_t *
 385 graph_add_vertex(const char *name)
 386 {
 387         int id;
 388         graph_vertex_t *v;
 389         void *p;
 390         uu_list_index_t idx;
 391 
 392         assert(MUTEX_HELD(&dgraph_lock));
 393 
 394         id = dict_insert(name);
 395 
 396         v = startd_zalloc(sizeof (*v));
 397 
 398         v->gv_id = id;
 399 
 400         v->gv_name = startd_alloc(strlen(name) + 1);
 401         (void) strcpy(v->gv_name, name);
 402 
 403         v->gv_dependencies = startd_list_create(graph_edge_pool, v, 0);
 404         v->gv_dependents = startd_list_create(graph_edge_pool, v, 0);
 405 
 406         p = uu_list_find(dgraph, &id, NULL, &idx);
 407         assert(p == NULL);
 408 
 409         uu_list_node_init(v, &v->gv_link, graph_vertex_pool);
 410         uu_list_insert(dgraph, v, idx);
 411 
 412         return (v);
 413 }
 414 
 415 /*
 416  * Removes v from the graph and frees it.  The graph should be locked by this
 417  * thread, and v should have no edges associated with it.
 418  */
 419 static void
 420 graph_remove_vertex(graph_vertex_t *v)
 421 {
 422         assert(MUTEX_HELD(&dgraph_lock));
 423 
 424         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 425         assert(uu_list_numnodes(v->gv_dependents) == 0);
 426         assert(v->gv_refs == 0);
 427 
 428         startd_free(v->gv_name, strlen(v->gv_name) + 1);
 429         uu_list_destroy(v->gv_dependencies);
 430         uu_list_destroy(v->gv_dependents);
 431         uu_list_remove(dgraph, v);
 432 
 433         startd_free(v, sizeof (graph_vertex_t));
 434 }
 435 
 436 static void
 437 graph_add_edge(graph_vertex_t *fv, graph_vertex_t *tv)
 438 {
 439         graph_edge_t *e, *re;
 440         int r;
 441 
 442         assert(MUTEX_HELD(&dgraph_lock));
 443 
 444         e = startd_alloc(sizeof (graph_edge_t));
 445         re = startd_alloc(sizeof (graph_edge_t));
 446 
 447         e->ge_parent = fv;
 448         e->ge_vertex = tv;
 449 
 450         re->ge_parent = tv;
 451         re->ge_vertex = fv;
 452 
 453         uu_list_node_init(e, &e->ge_link, graph_edge_pool);
 454         r = uu_list_insert_before(fv->gv_dependencies, NULL, e);
 455         assert(r == 0);
 456 
 457         uu_list_node_init(re, &re->ge_link, graph_edge_pool);
 458         r = uu_list_insert_before(tv->gv_dependents, NULL, re);
 459         assert(r == 0);
 460 }
 461 
 462 static void
 463 graph_remove_edge(graph_vertex_t *v, graph_vertex_t *dv)
 464 {
 465         graph_edge_t *e;
 466 
 467         for (e = uu_list_first(v->gv_dependencies);
 468             e != NULL;
 469             e = uu_list_next(v->gv_dependencies, e)) {
 470                 if (e->ge_vertex == dv) {
 471                         uu_list_remove(v->gv_dependencies, e);
 472                         startd_free(e, sizeof (graph_edge_t));
 473                         break;
 474                 }
 475         }
 476 
 477         for (e = uu_list_first(dv->gv_dependents);
 478             e != NULL;
 479             e = uu_list_next(dv->gv_dependents, e)) {
 480                 if (e->ge_vertex == v) {
 481                         uu_list_remove(dv->gv_dependents, e);
 482                         startd_free(e, sizeof (graph_edge_t));
 483                         break;
 484                 }
 485         }
 486 }
 487 
 488 static void
 489 remove_inst_vertex(graph_vertex_t *v)
 490 {
 491         graph_edge_t *e;
 492         graph_vertex_t *sv;
 493         int i;
 494 
 495         assert(MUTEX_HELD(&dgraph_lock));
 496         assert(uu_list_numnodes(v->gv_dependents) == 1);
 497         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 498         assert(v->gv_refs == 0);
 499         assert((v->gv_flags & GV_CONFIGURED) == 0);
 500 
 501         e = uu_list_first(v->gv_dependents);
 502         sv = e->ge_vertex;
 503         graph_remove_edge(sv, v);
 504 
 505         for (i = 0; up_svcs[i] != NULL; ++i) {
 506                 if (up_svcs_p[i] == v)
 507                         up_svcs_p[i] = NULL;
 508         }
 509 
 510         if (manifest_import_p == v)
 511                 manifest_import_p = NULL;
 512 
 513         graph_remove_vertex(v);
 514 
 515         if (uu_list_numnodes(sv->gv_dependencies) == 0 &&
 516             uu_list_numnodes(sv->gv_dependents) == 0 &&
 517             sv->gv_refs == 0)
 518                 graph_remove_vertex(sv);
 519 }
 520 
 521 static void
 522 graph_walk_dependents(graph_vertex_t *v, void (*func)(graph_vertex_t *, void *),
 523     void *arg)
 524 {
 525         graph_edge_t *e;
 526 
 527         for (e = uu_list_first(v->gv_dependents);
 528             e != NULL;
 529             e = uu_list_next(v->gv_dependents, e))
 530                 func(e->ge_vertex, arg);
 531 }
 532 
 533 static void
 534 graph_walk_dependencies(graph_vertex_t *v,
 535     void (*func)(graph_vertex_t *, void *), void *arg)
 536 {
 537         graph_edge_t *e;
 538 
 539         assert(MUTEX_HELD(&dgraph_lock));
 540 
 541         for (e = uu_list_first(v->gv_dependencies);
 542             e != NULL;
 543             e = uu_list_next(v->gv_dependencies, e)) {
 544 
 545                 func(e->ge_vertex, arg);
 546         }
 547 }
 548 
 549 /*
 550  * Generic graph walking function.
 551  *
 552  * Given a vertex, this function will walk either dependencies
 553  * (WALK_DEPENDENCIES) or dependents (WALK_DEPENDENTS) of a vertex recursively
 554  * for the entire graph.  It will avoid cycles and never visit the same vertex
 555  * twice.
 556  *
 557  * We avoid traversing exclusion dependencies, because they are allowed to
 558  * create cycles in the graph.  When propagating satisfiability, there is no
 559  * need to walk exclusion dependencies because exclude_all_satisfied() doesn't
 560  * test for satisfiability.
 561  *
 562  * The walker takes two callbacks.  The first is called before examining the
 563  * dependents of each vertex.  The second is called on each vertex after
 564  * examining its dependents.  This allows is_path_to() to construct a path only
 565  * after the target vertex has been found.
 566  */
 567 typedef enum {
 568         WALK_DEPENDENTS,
 569         WALK_DEPENDENCIES
 570 } graph_walk_dir_t;
 571 
 572 typedef int (*graph_walk_cb_t)(graph_vertex_t *, void *);
 573 
 574 typedef struct graph_walk_info {
 575         graph_walk_dir_t        gi_dir;
 576         uchar_t                 *gi_visited;    /* vertex bitmap */
 577         int                     (*gi_pre)(graph_vertex_t *, void *);
 578         void                    (*gi_post)(graph_vertex_t *, void *);
 579         void                    *gi_arg;        /* callback arg */
 580         int                     gi_ret;         /* return value */
 581 } graph_walk_info_t;
 582 
 583 static int
 584 graph_walk_recurse(graph_edge_t *e, graph_walk_info_t *gip)
 585 {
 586         uu_list_t *list;
 587         int r;
 588         graph_vertex_t *v = e->ge_vertex;
 589         int i;
 590         uint_t b;
 591 
 592         i = v->gv_id / 8;
 593         b = 1 << (v->gv_id % 8);
 594 
 595         /*
 596          * Check to see if we've visited this vertex already.
 597          */
 598         if (gip->gi_visited[i] & b)
 599                 return (UU_WALK_NEXT);
 600 
 601         gip->gi_visited[i] |= b;
 602 
 603         /*
 604          * Don't follow exclusions.
 605          */
 606         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
 607                 return (UU_WALK_NEXT);
 608 
 609         /*
 610          * Call pre-visit callback.  If this doesn't terminate the walk,
 611          * continue search.
 612          */
 613         if ((gip->gi_ret = gip->gi_pre(v, gip->gi_arg)) == UU_WALK_NEXT) {
 614                 /*
 615                  * Recurse using appropriate list.
 616                  */
 617                 if (gip->gi_dir == WALK_DEPENDENTS)
 618                         list = v->gv_dependents;
 619                 else
 620                         list = v->gv_dependencies;
 621 
 622                 r = uu_list_walk(list, (uu_walk_fn_t *)graph_walk_recurse,
 623                     gip, 0);
 624                 assert(r == 0);
 625         }
 626 
 627         /*
 628          * Callbacks must return either UU_WALK_NEXT or UU_WALK_DONE.
 629          */
 630         assert(gip->gi_ret == UU_WALK_NEXT || gip->gi_ret == UU_WALK_DONE);
 631 
 632         /*
 633          * If given a post-callback, call the function for every vertex.
 634          */
 635         if (gip->gi_post != NULL)
 636                 (void) gip->gi_post(v, gip->gi_arg);
 637 
 638         /*
 639          * Preserve the callback's return value.  If the callback returns
 640          * UU_WALK_DONE, then we propagate that to the caller in order to
 641          * terminate the walk.
 642          */
 643         return (gip->gi_ret);
 644 }
 645 
 646 static void
 647 graph_walk(graph_vertex_t *v, graph_walk_dir_t dir,
 648     int (*pre)(graph_vertex_t *, void *),
 649     void (*post)(graph_vertex_t *, void *), void *arg)
 650 {
 651         graph_walk_info_t gi;
 652         graph_edge_t fake;
 653         size_t sz = dictionary->dict_new_id / 8 + 1;
 654 
 655         gi.gi_visited = startd_zalloc(sz);
 656         gi.gi_pre = pre;
 657         gi.gi_post = post;
 658         gi.gi_arg = arg;
 659         gi.gi_dir = dir;
 660         gi.gi_ret = 0;
 661 
 662         /*
 663          * Fake up an edge for the first iteration
 664          */
 665         fake.ge_vertex = v;
 666         (void) graph_walk_recurse(&fake, &gi);
 667 
 668         startd_free(gi.gi_visited, sz);
 669 }
 670 
 671 typedef struct child_search {
 672         int     id;             /* id of vertex to look for */
 673         uint_t  depth;          /* recursion depth */
 674         /*
 675          * While the vertex is not found, path is NULL.  After the search, if
 676          * the vertex was found then path should point to a -1-terminated
 677          * array of vertex id's which constitute the path to the vertex.
 678          */
 679         int     *path;
 680 } child_search_t;
 681 
 682 static int
 683 child_pre(graph_vertex_t *v, void *arg)
 684 {
 685         child_search_t *cs = arg;
 686 
 687         cs->depth++;
 688 
 689         if (v->gv_id == cs->id) {
 690                 cs->path = startd_alloc((cs->depth + 1) * sizeof (int));
 691                 cs->path[cs->depth] = -1;
 692                 return (UU_WALK_DONE);
 693         }
 694 
 695         return (UU_WALK_NEXT);
 696 }
 697 
 698 static void
 699 child_post(graph_vertex_t *v, void *arg)
 700 {
 701         child_search_t *cs = arg;
 702 
 703         cs->depth--;
 704 
 705         if (cs->path != NULL)
 706                 cs->path[cs->depth] = v->gv_id;
 707 }
 708 
 709 /*
 710  * Look for a path from from to to.  If one exists, returns a pointer to
 711  * a NULL-terminated array of pointers to the vertices along the path.  If
 712  * there is no path, returns NULL.
 713  */
 714 static int *
 715 is_path_to(graph_vertex_t *from, graph_vertex_t *to)
 716 {
 717         child_search_t cs;
 718 
 719         cs.id = to->gv_id;
 720         cs.depth = 0;
 721         cs.path = NULL;
 722 
 723         graph_walk(from, WALK_DEPENDENCIES, child_pre, child_post, &cs);
 724 
 725         return (cs.path);
 726 }
 727 
 728 /*
 729  * Given an array of int's as returned by is_path_to, allocates a string of
 730  * their names joined by newlines.  Returns the size of the allocated buffer
 731  * in *sz and frees path.
 732  */
 733 static void
 734 path_to_str(int *path, char **cpp, size_t *sz)
 735 {
 736         int i;
 737         graph_vertex_t *v;
 738         size_t allocd, new_allocd;
 739         char *new, *name;
 740 
 741         assert(MUTEX_HELD(&dgraph_lock));
 742         assert(path[0] != -1);
 743 
 744         allocd = 1;
 745         *cpp = startd_alloc(1);
 746         (*cpp)[0] = '\0';
 747 
 748         for (i = 0; path[i] != -1; ++i) {
 749                 name = NULL;
 750 
 751                 v = vertex_get_by_id(path[i]);
 752 
 753                 if (v == NULL)
 754                         name = "<deleted>";
 755                 else if (v->gv_type == GVT_INST || v->gv_type == GVT_SVC)
 756                         name = v->gv_name;
 757 
 758                 if (name != NULL) {
 759                         new_allocd = allocd + strlen(name) + 1;
 760                         new = startd_alloc(new_allocd);
 761                         (void) strcpy(new, *cpp);
 762                         (void) strcat(new, name);
 763                         (void) strcat(new, "\n");
 764 
 765                         startd_free(*cpp, allocd);
 766 
 767                         *cpp = new;
 768                         allocd = new_allocd;
 769                 }
 770         }
 771 
 772         startd_free(path, sizeof (int) * (i + 1));
 773 
 774         *sz = allocd;
 775 }
 776 
 777 
 778 /*
 779  * This function along with run_sulogin() implements an exclusion relationship
 780  * between system/console-login and sulogin.  run_sulogin() will fail if
 781  * system/console-login is online, and the graph engine should call
 782  * graph_clogin_start() to bring system/console-login online, which defers the
 783  * start if sulogin is running.
 784  */
 785 static void
 786 graph_clogin_start(graph_vertex_t *v)
 787 {
 788         assert(MUTEX_HELD(&dgraph_lock));
 789 
 790         if (sulogin_running)
 791                 console_login_ready = B_TRUE;
 792         else
 793                 vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 794 }
 795 
 796 static void
 797 graph_su_start(graph_vertex_t *v)
 798 {
 799         /*
 800          * /etc/inittab used to have the initial /sbin/rcS as a 'sysinit'
 801          * entry with a runlevel of 'S', before jumping to the final
 802          * target runlevel (as set in initdefault).  We mimic that legacy
 803          * behavior here.
 804          */
 805         utmpx_set_runlevel('S', '0', B_FALSE);
 806         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 807 }
 808 
 809 static void
 810 graph_post_su_online(void)
 811 {
 812         graph_runlevel_changed('S', 1);
 813 }
 814 
 815 static void
 816 graph_post_su_disable(void)
 817 {
 818         graph_runlevel_changed('S', 0);
 819 }
 820 
 821 static void
 822 graph_post_mu_online(void)
 823 {
 824         graph_runlevel_changed('2', 1);
 825 }
 826 
 827 static void
 828 graph_post_mu_disable(void)
 829 {
 830         graph_runlevel_changed('2', 0);
 831 }
 832 
 833 static void
 834 graph_post_mus_online(void)
 835 {
 836         graph_runlevel_changed('3', 1);
 837 }
 838 
 839 static void
 840 graph_post_mus_disable(void)
 841 {
 842         graph_runlevel_changed('3', 0);
 843 }
 844 
 845 static struct special_vertex_info {
 846         const char      *name;
 847         void            (*start_f)(graph_vertex_t *);
 848         void            (*post_online_f)(void);
 849         void            (*post_disable_f)(void);
 850 } special_vertices[] = {
 851         { CONSOLE_LOGIN_FMRI, graph_clogin_start, NULL, NULL },
 852         { SCF_MILESTONE_SINGLE_USER, graph_su_start,
 853             graph_post_su_online, graph_post_su_disable },
 854         { SCF_MILESTONE_MULTI_USER, NULL,
 855             graph_post_mu_online, graph_post_mu_disable },
 856         { SCF_MILESTONE_MULTI_USER_SERVER, NULL,
 857             graph_post_mus_online, graph_post_mus_disable },
 858         { NULL },
 859 };
 860 
 861 
 862 void
 863 vertex_send_event(graph_vertex_t *v, restarter_event_type_t e)
 864 {
 865         switch (e) {
 866         case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
 867                 assert(v->gv_state == RESTARTER_STATE_UNINIT);
 868 
 869                 MUTEX_LOCK(&st->st_load_lock);
 870                 st->st_load_instances++;
 871                 MUTEX_UNLOCK(&st->st_load_lock);
 872                 break;
 873 
 874         case RESTARTER_EVENT_TYPE_ENABLE:
 875                 log_framework(LOG_DEBUG, "Enabling %s.\n", v->gv_name);
 876                 assert(v->gv_state == RESTARTER_STATE_UNINIT ||
 877                     v->gv_state == RESTARTER_STATE_DISABLED ||
 878                     v->gv_state == RESTARTER_STATE_MAINT);
 879                 break;
 880 
 881         case RESTARTER_EVENT_TYPE_DISABLE:
 882         case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
 883                 log_framework(LOG_DEBUG, "Disabling %s.\n", v->gv_name);
 884                 assert(v->gv_state != RESTARTER_STATE_DISABLED);
 885                 break;
 886 
 887         case RESTARTER_EVENT_TYPE_STOP_RESET:
 888         case RESTARTER_EVENT_TYPE_STOP:
 889                 log_framework(LOG_DEBUG, "Stopping %s.\n", v->gv_name);
 890                 assert(v->gv_state == RESTARTER_STATE_DEGRADED ||
 891                     v->gv_state == RESTARTER_STATE_ONLINE);
 892                 break;
 893 
 894         case RESTARTER_EVENT_TYPE_START:
 895                 log_framework(LOG_DEBUG, "Starting %s.\n", v->gv_name);
 896                 assert(v->gv_state == RESTARTER_STATE_OFFLINE);
 897                 break;
 898 
 899         case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
 900         case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
 901         case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
 902         case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
 903         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
 904         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
 905         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
 906         case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
 907         case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
 908                 break;
 909 
 910         default:
 911 #ifndef NDEBUG
 912                 uu_warn("%s:%d: Bad event %d.\n", __FILE__, __LINE__, e);
 913 #endif
 914                 abort();
 915         }
 916 
 917         restarter_protocol_send_event(v->gv_name, v->gv_restarter_channel, e,
 918             v->gv_reason);
 919 }
 920 
 921 static void
 922 graph_unset_restarter(graph_vertex_t *v)
 923 {
 924         assert(MUTEX_HELD(&dgraph_lock));
 925         assert(v->gv_flags & GV_CONFIGURED);
 926 
 927         vertex_send_event(v, RESTARTER_EVENT_TYPE_REMOVE_INSTANCE);
 928 
 929         if (v->gv_restarter_id != -1) {
 930                 graph_vertex_t *rv;
 931 
 932                 rv = vertex_get_by_id(v->gv_restarter_id);
 933                 graph_remove_edge(v, rv);
 934         }
 935 
 936         v->gv_restarter_id = -1;
 937         v->gv_restarter_channel = NULL;
 938 }
 939 
 940 /*
 941  * Return VERTEX_REMOVED when the vertex passed in argument is deleted from the
 942  * dgraph otherwise return VERTEX_INUSE.
 943  */
 944 static int
 945 free_if_unrefed(graph_vertex_t *v)
 946 {
 947         assert(MUTEX_HELD(&dgraph_lock));
 948 
 949         if (v->gv_refs > 0)
 950                 return (VERTEX_INUSE);
 951 
 952         if (v->gv_type == GVT_SVC &&
 953             uu_list_numnodes(v->gv_dependents) == 0 &&
 954             uu_list_numnodes(v->gv_dependencies) == 0) {
 955                 graph_remove_vertex(v);
 956                 return (VERTEX_REMOVED);
 957         } else if (v->gv_type == GVT_INST &&
 958             (v->gv_flags & GV_CONFIGURED) == 0 &&
 959             uu_list_numnodes(v->gv_dependents) == 1 &&
 960             uu_list_numnodes(v->gv_dependencies) == 0) {
 961                 remove_inst_vertex(v);
 962                 return (VERTEX_REMOVED);
 963         }
 964 
 965         return (VERTEX_INUSE);
 966 }
 967 
 968 static void
 969 delete_depgroup(graph_vertex_t *v)
 970 {
 971         graph_edge_t *e;
 972         graph_vertex_t *dv;
 973 
 974         assert(MUTEX_HELD(&dgraph_lock));
 975         assert(v->gv_type == GVT_GROUP);
 976         assert(uu_list_numnodes(v->gv_dependents) == 0);
 977 
 978         while ((e = uu_list_first(v->gv_dependencies)) != NULL) {
 979                 dv = e->ge_vertex;
 980 
 981                 graph_remove_edge(v, dv);
 982 
 983                 switch (dv->gv_type) {
 984                 case GVT_INST:          /* instance dependency */
 985                 case GVT_SVC:           /* service dependency */
 986                         (void) free_if_unrefed(dv);
 987                         break;
 988 
 989                 case GVT_FILE:          /* file dependency */
 990                         assert(uu_list_numnodes(dv->gv_dependencies) == 0);
 991                         if (uu_list_numnodes(dv->gv_dependents) == 0)
 992                                 graph_remove_vertex(dv);
 993                         break;
 994 
 995                 default:
 996 #ifndef NDEBUG
 997                         uu_warn("%s:%d: Unexpected node type %d", __FILE__,
 998                             __LINE__, dv->gv_type);
 999 #endif
1000                         abort();
1001                 }
1002         }
1003 
1004         graph_remove_vertex(v);
1005 }
1006 
1007 static int
1008 delete_instance_deps_cb(graph_edge_t *e, void **ptrs)
1009 {
1010         graph_vertex_t *v = ptrs[0];
1011         boolean_t delete_restarter_dep = (boolean_t)ptrs[1];
1012         graph_vertex_t *dv;
1013 
1014         dv = e->ge_vertex;
1015 
1016         /*
1017          * We have four possibilities here:
1018          *   - GVT_INST: restarter
1019          *   - GVT_GROUP - GVT_INST: instance dependency
1020          *   - GVT_GROUP - GVT_SVC - GV_INST: service dependency
1021          *   - GVT_GROUP - GVT_FILE: file dependency
1022          */
1023         switch (dv->gv_type) {
1024         case GVT_INST:  /* restarter */
1025                 assert(dv->gv_id == v->gv_restarter_id);
1026                 if (delete_restarter_dep)
1027                         graph_remove_edge(v, dv);
1028                 break;
1029 
1030         case GVT_GROUP: /* pg dependency */
1031                 graph_remove_edge(v, dv);
1032                 delete_depgroup(dv);
1033                 break;
1034 
1035         case GVT_FILE:
1036                 /* These are currently not direct dependencies */
1037 
1038         default:
1039 #ifndef NDEBUG
1040                 uu_warn("%s:%d: Bad vertex type %d.\n", __FILE__, __LINE__,
1041                     dv->gv_type);
1042 #endif
1043                 abort();
1044         }
1045 
1046         return (UU_WALK_NEXT);
1047 }
1048 
1049 static void
1050 delete_instance_dependencies(graph_vertex_t *v, boolean_t delete_restarter_dep)
1051 {
1052         void *ptrs[2];
1053         int r;
1054 
1055         assert(MUTEX_HELD(&dgraph_lock));
1056         assert(v->gv_type == GVT_INST);
1057 
1058         ptrs[0] = v;
1059         ptrs[1] = (void *)delete_restarter_dep;
1060 
1061         r = uu_list_walk(v->gv_dependencies,
1062             (uu_walk_fn_t *)delete_instance_deps_cb, &ptrs, UU_WALK_ROBUST);
1063         assert(r == 0);
1064 }
1065 
1066 /*
1067  * int graph_insert_vertex_unconfigured()
1068  *   Insert a vertex without sending any restarter events. If the vertex
1069  *   already exists or creation is successful, return a pointer to it in *vp.
1070  *
1071  *   If type is not GVT_GROUP, dt can remain unset.
1072  *
1073  *   Returns 0, EEXIST, or EINVAL if the arguments are invalid (i.e., fmri
1074  *   doesn't agree with type, or type doesn't agree with dt).
1075  */
1076 static int
1077 graph_insert_vertex_unconfigured(const char *fmri, gv_type_t type,
1078     depgroup_type_t dt, restarter_error_t rt, graph_vertex_t **vp)
1079 {
1080         int r;
1081         int i;
1082 
1083         assert(MUTEX_HELD(&dgraph_lock));
1084 
1085         switch (type) {
1086         case GVT_SVC:
1087         case GVT_INST:
1088                 if (strncmp(fmri, "svc:", sizeof ("svc:") - 1) != 0)
1089                         return (EINVAL);
1090                 break;
1091 
1092         case GVT_FILE:
1093                 if (strncmp(fmri, "file:", sizeof ("file:") - 1) != 0)
1094                         return (EINVAL);
1095                 break;
1096 
1097         case GVT_GROUP:
1098                 if (dt <= 0 || rt < 0)
1099                         return (EINVAL);
1100                 break;
1101 
1102         default:
1103 #ifndef NDEBUG
1104                 uu_warn("%s:%d: Unknown type %d.\n", __FILE__, __LINE__, type);
1105 #endif
1106                 abort();
1107         }
1108 
1109         *vp = vertex_get_by_name(fmri);
1110         if (*vp != NULL)
1111                 return (EEXIST);
1112 
1113         *vp = graph_add_vertex(fmri);
1114 
1115         (*vp)->gv_type = type;
1116         (*vp)->gv_depgroup = dt;
1117         (*vp)->gv_restart = rt;
1118 
1119         (*vp)->gv_flags = 0;
1120         (*vp)->gv_state = RESTARTER_STATE_NONE;
1121 
1122         for (i = 0; special_vertices[i].name != NULL; ++i) {
1123                 if (strcmp(fmri, special_vertices[i].name) == 0) {
1124                         (*vp)->gv_start_f = special_vertices[i].start_f;
1125                         (*vp)->gv_post_online_f =
1126                             special_vertices[i].post_online_f;
1127                         (*vp)->gv_post_disable_f =
1128                             special_vertices[i].post_disable_f;
1129                         break;
1130                 }
1131         }
1132 
1133         (*vp)->gv_restarter_id = -1;
1134         (*vp)->gv_restarter_channel = 0;
1135 
1136         if (type == GVT_INST) {
1137                 char *sfmri;
1138                 graph_vertex_t *sv;
1139 
1140                 sfmri = inst_fmri_to_svc_fmri(fmri);
1141                 sv = vertex_get_by_name(sfmri);
1142                 if (sv == NULL) {
1143                         r = graph_insert_vertex_unconfigured(sfmri, GVT_SVC, 0,
1144                             0, &sv);
1145                         assert(r == 0);
1146                 }
1147                 startd_free(sfmri, max_scf_fmri_size);
1148 
1149                 graph_add_edge(sv, *vp);
1150         }
1151 
1152         /*
1153          * If this vertex is in the subgraph, mark it as so, for both
1154          * GVT_INST and GVT_SERVICE verteces.
1155          * A GVT_SERVICE vertex can only be in the subgraph if another instance
1156          * depends on it, in which case it's already been added to the graph
1157          * and marked as in the subgraph (by refresh_vertex()).  If a
1158          * GVT_SERVICE vertex was freshly added (by the code above), it means
1159          * that it has no dependents, and cannot be in the subgraph.
1160          * Regardless of this, we still check that gv_flags includes
1161          * GV_INSUBGRAPH in the event that future behavior causes the above
1162          * code to add a GVT_SERVICE vertex which should be in the subgraph.
1163          */
1164 
1165         (*vp)->gv_flags |= (should_be_in_subgraph(*vp)? GV_INSUBGRAPH : 0);
1166 
1167         return (0);
1168 }
1169 
1170 /*
1171  * Returns 0 on success or ELOOP if the dependency would create a cycle.
1172  */
1173 static int
1174 graph_insert_dependency(graph_vertex_t *fv, graph_vertex_t *tv, int **pathp)
1175 {
1176         hrtime_t now;
1177 
1178         assert(MUTEX_HELD(&dgraph_lock));
1179 
1180         /* cycle detection */
1181         now = gethrtime();
1182 
1183         /* Don't follow exclusions. */
1184         if (!(fv->gv_type == GVT_GROUP &&
1185             fv->gv_depgroup == DEPGRP_EXCLUDE_ALL)) {
1186                 *pathp = is_path_to(tv, fv);
1187                 if (*pathp)
1188                         return (ELOOP);
1189         }
1190 
1191         dep_cycle_ns += gethrtime() - now;
1192         ++dep_inserts;
1193         now = gethrtime();
1194 
1195         graph_add_edge(fv, tv);
1196 
1197         dep_insert_ns += gethrtime() - now;
1198 
1199         /* Check if the dependency adds the "to" vertex to the subgraph */
1200         tv->gv_flags |= (should_be_in_subgraph(tv) ? GV_INSUBGRAPH : 0);
1201 
1202         return (0);
1203 }
1204 
1205 static int
1206 inst_running(graph_vertex_t *v)
1207 {
1208         assert(v->gv_type == GVT_INST);
1209 
1210         if (v->gv_state == RESTARTER_STATE_ONLINE ||
1211             v->gv_state == RESTARTER_STATE_DEGRADED)
1212                 return (1);
1213 
1214         return (0);
1215 }
1216 
1217 /*
1218  * The dependency evaluation functions return
1219  *   1 - dependency satisfied
1220  *   0 - dependency unsatisfied
1221  *   -1 - dependency unsatisfiable (without administrator intervention)
1222  *
1223  * The functions also take a boolean satbility argument.  When true, the
1224  * functions may recurse in order to determine satisfiability.
1225  */
1226 static int require_any_satisfied(graph_vertex_t *, boolean_t);
1227 static int dependency_satisfied(graph_vertex_t *, boolean_t);
1228 
1229 /*
1230  * A require_all dependency is unsatisfied if any elements are unsatisfied.  It
1231  * is unsatisfiable if any elements are unsatisfiable.
1232  */
1233 static int
1234 require_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1235 {
1236         graph_edge_t *edge;
1237         int i;
1238         boolean_t any_unsatisfied;
1239 
1240         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1241                 return (1);
1242 
1243         any_unsatisfied = B_FALSE;
1244 
1245         for (edge = uu_list_first(groupv->gv_dependencies);
1246             edge != NULL;
1247             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1248                 i = dependency_satisfied(edge->ge_vertex, satbility);
1249                 if (i == 1)
1250                         continue;
1251 
1252                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1253                     "require_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1254                     edge->ge_vertex->gv_name, i == 0 ? "ed" : "able");
1255 
1256                 if (!satbility)
1257                         return (0);
1258 
1259                 if (i == -1)
1260                         return (-1);
1261 
1262                 any_unsatisfied = B_TRUE;
1263         }
1264 
1265         return (any_unsatisfied ? 0 : 1);
1266 }
1267 
1268 /*
1269  * A require_any dependency is satisfied if any element is satisfied.  It is
1270  * satisfiable if any element is satisfiable.
1271  */
1272 static int
1273 require_any_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1274 {
1275         graph_edge_t *edge;
1276         int s;
1277         boolean_t satisfiable;
1278 
1279         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1280                 return (1);
1281 
1282         satisfiable = B_FALSE;
1283 
1284         for (edge = uu_list_first(groupv->gv_dependencies);
1285             edge != NULL;
1286             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1287                 s = dependency_satisfied(edge->ge_vertex, satbility);
1288 
1289                 if (s == 1)
1290                         return (1);
1291 
1292                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1293                     "require_any(%s): %s is unsatisfi%s.\n",
1294                     groupv->gv_name, edge->ge_vertex->gv_name,
1295                     s == 0 ? "ed" : "able");
1296 
1297                 if (satbility && s == 0)
1298                         satisfiable = B_TRUE;
1299         }
1300 
1301         return (!satbility || satisfiable ? 0 : -1);
1302 }
1303 
1304 /*
1305  * An optional_all dependency only considers elements which are configured,
1306  * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1307  * is unsatisfied.
1308  *
1309  * Offline dependencies which are waiting for a dependency to come online are
1310  * unsatisfied.  Offline dependences which cannot possibly come online
1311  * (unsatisfiable) are always considered satisfied.
1312  */
1313 static int
1314 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1315 {
1316         graph_edge_t *edge;
1317         graph_vertex_t *v;
1318         boolean_t any_qualified;
1319         boolean_t any_unsatisfied;
1320         int i;
1321 
1322         any_qualified = B_FALSE;
1323         any_unsatisfied = B_FALSE;
1324 
1325         for (edge = uu_list_first(groupv->gv_dependencies);
1326             edge != NULL;
1327             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1328                 v = edge->ge_vertex;
1329 
1330                 switch (v->gv_type) {
1331                 case GVT_INST:
1332                         /* Skip missing or disabled instances */
1333                         if ((v->gv_flags & (GV_CONFIGURED | GV_ENABLED)) !=
1334                             (GV_CONFIGURED | GV_ENABLED))
1335                                 continue;
1336 
1337                         if (v->gv_state == RESTARTER_STATE_MAINT)
1338                                 continue;
1339 
1340                         if (v->gv_flags & GV_TOOFFLINE)
1341                                 continue;
1342 
1343                         any_qualified = B_TRUE;
1344                         if (v->gv_state == RESTARTER_STATE_OFFLINE) {
1345                                 /*
1346                                  * For offline dependencies, treat unsatisfiable
1347                                  * as satisfied.
1348                                  */
1349                                 i = dependency_satisfied(v, B_TRUE);
1350                                 if (i == -1)
1351                                         i = 1;
1352                         } else if (v->gv_state == RESTARTER_STATE_DISABLED) {
1353                                 /*
1354                                  * The service is enabled, but hasn't
1355                                  * transitioned out of disabled yet.  Treat it
1356                                  * as unsatisfied (not unsatisfiable).
1357                                  */
1358                                 i = 0;
1359                         } else {
1360                                 i = dependency_satisfied(v, satbility);
1361                         }
1362                         break;
1363 
1364                 case GVT_FILE:
1365                         any_qualified = B_TRUE;
1366                         i = dependency_satisfied(v, satbility);
1367 
1368                         break;
1369 
1370                 case GVT_SVC: {
1371                         boolean_t svc_any_qualified;
1372                         boolean_t svc_satisfied;
1373                         boolean_t svc_satisfiable;
1374                         graph_vertex_t *v2;
1375                         graph_edge_t *e2;
1376 
1377                         svc_any_qualified = B_FALSE;
1378                         svc_satisfied = B_FALSE;
1379                         svc_satisfiable = B_FALSE;
1380 
1381                         for (e2 = uu_list_first(v->gv_dependencies);
1382                             e2 != NULL;
1383                             e2 = uu_list_next(v->gv_dependencies, e2)) {
1384                                 v2 = e2->ge_vertex;
1385                                 assert(v2->gv_type == GVT_INST);
1386 
1387                                 if ((v2->gv_flags &
1388                                     (GV_CONFIGURED | GV_ENABLED)) !=
1389                                     (GV_CONFIGURED | GV_ENABLED))
1390                                         continue;
1391 
1392                                 if (v2->gv_state == RESTARTER_STATE_MAINT)
1393                                         continue;
1394 
1395                                 if (v2->gv_flags & GV_TOOFFLINE)
1396                                         continue;
1397 
1398                                 svc_any_qualified = B_TRUE;
1399 
1400                                 if (v2->gv_state == RESTARTER_STATE_OFFLINE) {
1401                                         /*
1402                                          * For offline dependencies, treat
1403                                          * unsatisfiable as satisfied.
1404                                          */
1405                                         i = dependency_satisfied(v2, B_TRUE);
1406                                         if (i == -1)
1407                                                 i = 1;
1408                                 } else if (v2->gv_state ==
1409                                     RESTARTER_STATE_DISABLED) {
1410                                         i = 0;
1411                                 } else {
1412                                         i = dependency_satisfied(v2, satbility);
1413                                 }
1414 
1415                                 if (i == 1) {
1416                                         svc_satisfied = B_TRUE;
1417                                         break;
1418                                 }
1419                                 if (i == 0)
1420                                         svc_satisfiable = B_TRUE;
1421                         }
1422 
1423                         if (!svc_any_qualified)
1424                                 continue;
1425                         any_qualified = B_TRUE;
1426                         if (svc_satisfied) {
1427                                 i = 1;
1428                         } else if (svc_satisfiable) {
1429                                 i = 0;
1430                         } else {
1431                                 i = -1;
1432                         }
1433                         break;
1434                 }
1435 
1436                 case GVT_GROUP:
1437                 default:
1438 #ifndef NDEBUG
1439                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1440                             __LINE__, v->gv_type);
1441 #endif
1442                         abort();
1443                 }
1444 
1445                 if (i == 1)
1446                         continue;
1447 
1448                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1449                     "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1450                     v->gv_name, i == 0 ? "ed" : "able");
1451 
1452                 if (!satbility)
1453                         return (0);
1454                 if (i == -1)
1455                         return (-1);
1456                 any_unsatisfied = B_TRUE;
1457         }
1458 
1459         if (!any_qualified)
1460                 return (1);
1461 
1462         return (any_unsatisfied ? 0 : 1);
1463 }
1464 
1465 /*
1466  * An exclude_all dependency is unsatisfied if any non-service element is
1467  * satisfied or any service instance which is configured, enabled, and not in
1468  * maintenance is satisfied.  Usually when unsatisfied, it is also
1469  * unsatisfiable.
1470  */
1471 #define LOG_EXCLUDE(u, v)                                               \
1472         log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,                   \
1473             "exclude_all(%s): %s is satisfied.\n",                      \
1474             (u)->gv_name, (v)->gv_name)
1475 
1476 /* ARGSUSED */
1477 static int
1478 exclude_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1479 {
1480         graph_edge_t *edge, *e2;
1481         graph_vertex_t *v, *v2;
1482 
1483         for (edge = uu_list_first(groupv->gv_dependencies);
1484             edge != NULL;
1485             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1486                 v = edge->ge_vertex;
1487 
1488                 switch (v->gv_type) {
1489                 case GVT_INST:
1490                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1491                                 continue;
1492 
1493                         switch (v->gv_state) {
1494                         case RESTARTER_STATE_ONLINE:
1495                         case RESTARTER_STATE_DEGRADED:
1496                                 LOG_EXCLUDE(groupv, v);
1497                                 return (v->gv_flags & GV_ENABLED ? -1 : 0);
1498 
1499                         case RESTARTER_STATE_OFFLINE:
1500                         case RESTARTER_STATE_UNINIT:
1501                                 LOG_EXCLUDE(groupv, v);
1502                                 return (0);
1503 
1504                         case RESTARTER_STATE_DISABLED:
1505                         case RESTARTER_STATE_MAINT:
1506                                 continue;
1507 
1508                         default:
1509 #ifndef NDEBUG
1510                                 uu_warn("%s:%d: Unexpected vertex state %d.\n",
1511                                     __FILE__, __LINE__, v->gv_state);
1512 #endif
1513                                 abort();
1514                         }
1515                         /* NOTREACHED */
1516 
1517                 case GVT_SVC:
1518                         break;
1519 
1520                 case GVT_FILE:
1521                         if (!file_ready(v))
1522                                 continue;
1523                         LOG_EXCLUDE(groupv, v);
1524                         return (-1);
1525 
1526                 case GVT_GROUP:
1527                 default:
1528 #ifndef NDEBUG
1529                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1530                             __LINE__, v->gv_type);
1531 #endif
1532                         abort();
1533                 }
1534 
1535                 /* v represents a service */
1536                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1537                         continue;
1538 
1539                 for (e2 = uu_list_first(v->gv_dependencies);
1540                     e2 != NULL;
1541                     e2 = uu_list_next(v->gv_dependencies, e2)) {
1542                         v2 = e2->ge_vertex;
1543                         assert(v2->gv_type == GVT_INST);
1544 
1545                         if ((v2->gv_flags & GV_CONFIGURED) == 0)
1546                                 continue;
1547 
1548                         switch (v2->gv_state) {
1549                         case RESTARTER_STATE_ONLINE:
1550                         case RESTARTER_STATE_DEGRADED:
1551                                 LOG_EXCLUDE(groupv, v2);
1552                                 return (v2->gv_flags & GV_ENABLED ? -1 : 0);
1553 
1554                         case RESTARTER_STATE_OFFLINE:
1555                         case RESTARTER_STATE_UNINIT:
1556                                 LOG_EXCLUDE(groupv, v2);
1557                                 return (0);
1558 
1559                         case RESTARTER_STATE_DISABLED:
1560                         case RESTARTER_STATE_MAINT:
1561                                 continue;
1562 
1563                         default:
1564 #ifndef NDEBUG
1565                                 uu_warn("%s:%d: Unexpected vertex type %d.\n",
1566                                     __FILE__, __LINE__, v2->gv_type);
1567 #endif
1568                                 abort();
1569                         }
1570                 }
1571         }
1572 
1573         return (1);
1574 }
1575 
1576 /*
1577  * int instance_satisfied()
1578  *   Determine if all the dependencies are satisfied for the supplied instance
1579  *   vertex. Return 1 if they are, 0 if they aren't, and -1 if they won't be
1580  *   without administrator intervention.
1581  */
1582 static int
1583 instance_satisfied(graph_vertex_t *v, boolean_t satbility)
1584 {
1585         assert(v->gv_type == GVT_INST);
1586         assert(!inst_running(v));
1587 
1588         return (require_all_satisfied(v, satbility));
1589 }
1590 
1591 /*
1592  * Decide whether v can satisfy a dependency.  v can either be a child of
1593  * a group vertex, or of an instance vertex.
1594  */
1595 static int
1596 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1597 {
1598         switch (v->gv_type) {
1599         case GVT_INST:
1600                 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1601                         if (v->gv_flags & GV_DEATHROW) {
1602                                 /*
1603                                  * A dependency on an instance with GV_DEATHROW
1604                                  * flag is always considered as satisfied.
1605                                  */
1606                                 return (1);
1607                         }
1608                         return (-1);
1609                 }
1610 
1611                 /*
1612                  * Any vertex with the GV_TOOFFLINE flag set is guaranteed
1613                  * to have its dependencies unsatisfiable.
1614                  */
1615                 if (v->gv_flags & GV_TOOFFLINE)
1616                         return (-1);
1617 
1618                 switch (v->gv_state) {
1619                 case RESTARTER_STATE_ONLINE:
1620                 case RESTARTER_STATE_DEGRADED:
1621                         return (1);
1622 
1623                 case RESTARTER_STATE_OFFLINE:
1624                         if (!satbility)
1625                                 return (0);
1626                         return (instance_satisfied(v, satbility) != -1 ?
1627                             0 : -1);
1628 
1629                 case RESTARTER_STATE_DISABLED:
1630                 case RESTARTER_STATE_MAINT:
1631                         return (-1);
1632 
1633                 case RESTARTER_STATE_UNINIT:
1634                         return (0);
1635 
1636                 default:
1637 #ifndef NDEBUG
1638                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
1639                             __FILE__, __LINE__, v->gv_state);
1640 #endif
1641                         abort();
1642                         /* NOTREACHED */
1643                 }
1644 
1645         case GVT_SVC:
1646                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1647                         return (-1);
1648                 return (require_any_satisfied(v, satbility));
1649 
1650         case GVT_FILE:
1651                 /* i.e., we assume files will not be automatically generated */
1652                 return (file_ready(v) ? 1 : -1);
1653 
1654         case GVT_GROUP:
1655                 break;
1656 
1657         default:
1658 #ifndef NDEBUG
1659                 uu_warn("%s:%d: Unexpected node type %d.\n", __FILE__, __LINE__,
1660                     v->gv_type);
1661 #endif
1662                 abort();
1663                 /* NOTREACHED */
1664         }
1665 
1666         switch (v->gv_depgroup) {
1667         case DEPGRP_REQUIRE_ANY:
1668                 return (require_any_satisfied(v, satbility));
1669 
1670         case DEPGRP_REQUIRE_ALL:
1671                 return (require_all_satisfied(v, satbility));
1672 
1673         case DEPGRP_OPTIONAL_ALL:
1674                 return (optional_all_satisfied(v, satbility));
1675 
1676         case DEPGRP_EXCLUDE_ALL:
1677                 return (exclude_all_satisfied(v, satbility));
1678 
1679         default:
1680 #ifndef NDEBUG
1681                 uu_warn("%s:%d: Unknown dependency grouping %d.\n", __FILE__,
1682                     __LINE__, v->gv_depgroup);
1683 #endif
1684                 abort();
1685         }
1686 }
1687 
1688 void
1689 graph_start_if_satisfied(graph_vertex_t *v)
1690 {
1691         if (v->gv_state == RESTARTER_STATE_OFFLINE &&
1692             instance_satisfied(v, B_FALSE) == 1) {
1693                 if (v->gv_start_f == NULL)
1694                         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
1695                 else
1696                         v->gv_start_f(v);
1697         }
1698 }
1699 
1700 /*
1701  * propagate_satbility()
1702  *
1703  * This function is used when the given vertex changes state in such a way that
1704  * one of its dependents may become unsatisfiable.  This happens when an
1705  * instance transitions between offline -> online, or from !running ->
1706  * maintenance, as well as when an instance is removed from the graph.
1707  *
1708  * We have to walk all the dependents, since optional_all dependencies several
1709  * levels up could become (un)satisfied, instead of unsatisfiable.  For example,
1710  *
1711  *      +-----+  optional_all  +-----+  require_all  +-----+
1712  *      |  A  |--------------->|  B  |-------------->|  C  |
1713  *      +-----+                +-----+               +-----+
1714  *
1715  *                                              offline -> maintenance
1716  *
1717  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1718  * an optional dependency, what was previously an unsatisfiable situation is now
1719  * satisfied (B will never come online, even though its state hasn't changed).
1720  *
1721  * Note that it's not necessary to continue examining dependents after reaching
1722  * an optional_all dependency.  It's not possible for an optional_all dependency
1723  * to change satisfiability without also coming online, in which case we get a
1724  * start event and propagation continues naturally.  However, it does no harm to
1725  * continue propagating satisfiability (as it is a relatively rare event), and
1726  * keeps the walker code simple and generic.
1727  */
1728 /*ARGSUSED*/
1729 static int
1730 satbility_cb(graph_vertex_t *v, void *arg)
1731 {
1732         if (v->gv_type == GVT_INST)
1733                 graph_start_if_satisfied(v);
1734 
1735         return (UU_WALK_NEXT);
1736 }
1737 
1738 static void
1739 propagate_satbility(graph_vertex_t *v)
1740 {
1741         graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1742 }
1743 
1744 static void propagate_stop(graph_vertex_t *, void *);
1745 
1746 /* ARGSUSED */
1747 static void
1748 propagate_start(graph_vertex_t *v, void *arg)
1749 {
1750         switch (v->gv_type) {
1751         case GVT_INST:
1752                 graph_start_if_satisfied(v);
1753                 break;
1754 
1755         case GVT_GROUP:
1756                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1757                         graph_walk_dependents(v, propagate_stop,
1758                             (void *)RERR_RESTART);
1759                         break;
1760                 }
1761                 /* FALLTHROUGH */
1762 
1763         case GVT_SVC:
1764                 graph_walk_dependents(v, propagate_start, NULL);
1765                 break;
1766 
1767         case GVT_FILE:
1768 #ifndef NDEBUG
1769                 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1770                     __FILE__, __LINE__);
1771 #endif
1772                 abort();
1773                 /* NOTREACHED */
1774 
1775         default:
1776 #ifndef NDEBUG
1777                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1778                     v->gv_type);
1779 #endif
1780                 abort();
1781         }
1782 }
1783 
1784 static void
1785 propagate_stop(graph_vertex_t *v, void *arg)
1786 {
1787         graph_edge_t *e;
1788         graph_vertex_t *svc;
1789         restarter_error_t err = (restarter_error_t)arg;
1790 
1791         switch (v->gv_type) {
1792         case GVT_INST:
1793                 /* Restarter */
1794                 if (err > RERR_NONE && inst_running(v)) {
1795                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1796                                 vertex_send_event(v,
1797                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1798                         } else {
1799                                 vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1800                         }
1801                 }
1802                 break;
1803 
1804         case GVT_SVC:
1805                 graph_walk_dependents(v, propagate_stop, arg);
1806                 break;
1807 
1808         case GVT_FILE:
1809 #ifndef NDEBUG
1810                 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1811                     __FILE__, __LINE__);
1812 #endif
1813                 abort();
1814                 /* NOTREACHED */
1815 
1816         case GVT_GROUP:
1817                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1818                         graph_walk_dependents(v, propagate_start, NULL);
1819                         break;
1820                 }
1821 
1822                 if (err == RERR_NONE || err > v->gv_restart)
1823                         break;
1824 
1825                 assert(uu_list_numnodes(v->gv_dependents) == 1);
1826                 e = uu_list_first(v->gv_dependents);
1827                 svc = e->ge_vertex;
1828 
1829                 if (inst_running(svc)) {
1830                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1831                                 vertex_send_event(svc,
1832                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1833                         } else {
1834                                 vertex_send_event(svc,
1835                                     RESTARTER_EVENT_TYPE_STOP);
1836                         }
1837                 }
1838                 break;
1839 
1840         default:
1841 #ifndef NDEBUG
1842                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1843                     v->gv_type);
1844 #endif
1845                 abort();
1846         }
1847 }
1848 
1849 void
1850 offline_vertex(graph_vertex_t *v)
1851 {
1852         scf_handle_t *h = libscf_handle_create_bound_loop();
1853         scf_instance_t *scf_inst = safe_scf_instance_create(h);
1854         scf_propertygroup_t *pg = safe_scf_pg_create(h);
1855         restarter_instance_state_t state, next_state;
1856         int r;
1857 
1858         assert(v->gv_type == GVT_INST);
1859 
1860         if (scf_inst == NULL)
1861                 bad_error("safe_scf_instance_create", scf_error());
1862         if (pg == NULL)
1863                 bad_error("safe_scf_pg_create", scf_error());
1864 
1865         /* if the vertex is already going offline, return */
1866 rep_retry:
1867         if (scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, scf_inst, NULL,
1868             NULL, SCF_DECODE_FMRI_EXACT) != 0) {
1869                 switch (scf_error()) {
1870                 case SCF_ERROR_CONNECTION_BROKEN:
1871                         libscf_handle_rebind(h);
1872                         goto rep_retry;
1873 
1874                 case SCF_ERROR_NOT_FOUND:
1875                         scf_pg_destroy(pg);
1876                         scf_instance_destroy(scf_inst);
1877                         (void) scf_handle_unbind(h);
1878                         scf_handle_destroy(h);
1879                         return;
1880                 }
1881                 uu_die("Can't decode FMRI %s: %s\n", v->gv_name,
1882                     scf_strerror(scf_error()));
1883         }
1884 
1885         r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
1886         if (r != 0) {
1887                 switch (scf_error()) {
1888                 case SCF_ERROR_CONNECTION_BROKEN:
1889                         libscf_handle_rebind(h);
1890                         goto rep_retry;
1891 
1892                 case SCF_ERROR_NOT_SET:
1893                 case SCF_ERROR_NOT_FOUND:
1894                         scf_pg_destroy(pg);
1895                         scf_instance_destroy(scf_inst);
1896                         (void) scf_handle_unbind(h);
1897                         scf_handle_destroy(h);
1898                         return;
1899 
1900                 default:
1901                         bad_error("scf_instance_get_pg", scf_error());
1902                 }
1903         } else {
1904                 r = libscf_read_states(pg, &state, &next_state);
1905                 if (r == 0 && (next_state == RESTARTER_STATE_OFFLINE ||
1906                     next_state == RESTARTER_STATE_DISABLED)) {
1907                         log_framework(LOG_DEBUG,
1908                             "%s: instance is already going down.\n",
1909                             v->gv_name);
1910                         scf_pg_destroy(pg);
1911                         scf_instance_destroy(scf_inst);
1912                         (void) scf_handle_unbind(h);
1913                         scf_handle_destroy(h);
1914                         return;
1915                 }
1916         }
1917 
1918         scf_pg_destroy(pg);
1919         scf_instance_destroy(scf_inst);
1920         (void) scf_handle_unbind(h);
1921         scf_handle_destroy(h);
1922 
1923         vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP_RESET);
1924 }
1925 
1926 /*
1927  * void graph_enable_by_vertex()
1928  *   If admin is non-zero, this is an administrative request for change
1929  *   of the enabled property.  Thus, send the ADMIN_DISABLE rather than
1930  *   a plain DISABLE restarter event.
1931  */
1932 void
1933 graph_enable_by_vertex(graph_vertex_t *vertex, int enable, int admin)
1934 {
1935         graph_vertex_t *v;
1936         int r;
1937 
1938         assert(MUTEX_HELD(&dgraph_lock));
1939         assert((vertex->gv_flags & GV_CONFIGURED));
1940 
1941         vertex->gv_flags = (vertex->gv_flags & ~GV_ENABLED) |
1942             (enable ? GV_ENABLED : 0);
1943 
1944         if (enable) {
1945                 if (vertex->gv_state != RESTARTER_STATE_OFFLINE &&
1946                     vertex->gv_state != RESTARTER_STATE_DEGRADED &&
1947                     vertex->gv_state != RESTARTER_STATE_ONLINE) {
1948                         /*
1949                          * In case the vertex was notified to go down,
1950                          * but now can return online, clear the _TOOFFLINE
1951                          * and _TODISABLE flags.
1952                          */
1953                         vertex->gv_flags &= ~GV_TOOFFLINE;
1954                         vertex->gv_flags &= ~GV_TODISABLE;
1955 
1956                         vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ENABLE);
1957                 }
1958 
1959                 /*
1960                  * Wait for state update from restarter before sending _START or
1961                  * _STOP.
1962                  */
1963 
1964                 return;
1965         }
1966 
1967         if (vertex->gv_state == RESTARTER_STATE_DISABLED)
1968                 return;
1969 
1970         if (!admin) {
1971                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_DISABLE);
1972 
1973                 /*
1974                  * Wait for state update from restarter before sending _START or
1975                  * _STOP.
1976                  */
1977 
1978                 return;
1979         }
1980 
1981         /*
1982          * If it is a DISABLE event requested by the administrator then we are
1983          * offlining the dependents first.
1984          */
1985 
1986         /*
1987          * Set GV_TOOFFLINE for the services we are offlining. We cannot
1988          * clear the GV_TOOFFLINE bits from all the services because
1989          * other DISABLE events might be handled at the same time.
1990          */
1991         vertex->gv_flags |= GV_TOOFFLINE;
1992 
1993         /* remember which vertex to disable... */
1994         vertex->gv_flags |= GV_TODISABLE;
1995 
1996         log_framework(LOG_DEBUG, "Marking in-subtree vertices before "
1997             "disabling %s.\n", vertex->gv_name);
1998 
1999         /* set GV_TOOFFLINE for its dependents */
2000         r = uu_list_walk(vertex->gv_dependents, (uu_walk_fn_t *)mark_subtree,
2001             NULL, 0);
2002         assert(r == 0);
2003 
2004         /* disable the instance now if there is nothing else to offline */
2005         if (insubtree_dependents_down(vertex) == B_TRUE) {
2006                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
2007                 return;
2008         }
2009 
2010         /*
2011          * This loop is similar to the one used for the graph reversal shutdown
2012          * and could be improved in term of performance for the subtree reversal
2013          * disable case.
2014          */
2015         for (v = uu_list_first(dgraph); v != NULL;
2016             v = uu_list_next(dgraph, v)) {
2017                 /* skip the vertex we are disabling for now */
2018                 if (v == vertex)
2019                         continue;
2020 
2021                 if (v->gv_type != GVT_INST ||
2022                     (v->gv_flags & GV_CONFIGURED) == 0 ||
2023                     (v->gv_flags & GV_ENABLED) == 0 ||
2024                     (v->gv_flags & GV_TOOFFLINE) == 0)
2025                         continue;
2026 
2027                 if ((v->gv_state != RESTARTER_STATE_ONLINE) &&
2028                     (v->gv_state != RESTARTER_STATE_DEGRADED)) {
2029                         /* continue if there is nothing to offline */
2030                         continue;
2031                 }
2032 
2033                 /*
2034                  * Instances which are up need to come down before we're
2035                  * done, but we can only offline the leaves here. An
2036                  * instance is a leaf when all its dependents are down.
2037                  */
2038                 if (insubtree_dependents_down(v) == B_TRUE) {
2039                         log_framework(LOG_DEBUG, "Offlining in-subtree "
2040                             "instance %s for %s.\n",
2041                             v->gv_name, vertex->gv_name);
2042                         offline_vertex(v);
2043                 }
2044         }
2045 }
2046 
2047 static int configure_vertex(graph_vertex_t *, scf_instance_t *);
2048 
2049 /*
2050  * Set the restarter for v to fmri_arg.  That is, make sure a vertex for
2051  * fmri_arg exists, make v depend on it, and send _ADD_INSTANCE for v.  If
2052  * v is already configured and fmri_arg indicates the current restarter, do
2053  * nothing.  If v is configured and fmri_arg is a new restarter, delete v's
2054  * dependency on the restarter, send _REMOVE_INSTANCE for v, and set the new
2055  * restarter.  Returns 0 on success, EINVAL if the FMRI is invalid,
2056  * ECONNABORTED if the repository connection is broken, and ELOOP
2057  * if the dependency would create a cycle.  In the last case, *pathp will
2058  * point to a -1-terminated array of ids which compose the path from v to
2059  * restarter_fmri.
2060  */
2061 int
2062 graph_change_restarter(graph_vertex_t *v, const char *fmri_arg, scf_handle_t *h,
2063     int **pathp)
2064 {
2065         char *restarter_fmri = NULL;
2066         graph_vertex_t *rv;
2067         int err;
2068         int id;
2069 
2070         assert(MUTEX_HELD(&dgraph_lock));
2071 
2072         if (fmri_arg[0] != '\0') {
2073                 err = fmri_canonify(fmri_arg, &restarter_fmri, B_TRUE);
2074                 if (err != 0) {
2075                         assert(err == EINVAL);
2076                         return (err);
2077                 }
2078         }
2079 
2080         if (restarter_fmri == NULL ||
2081             strcmp(restarter_fmri, SCF_SERVICE_STARTD) == 0) {
2082                 if (v->gv_flags & GV_CONFIGURED) {
2083                         if (v->gv_restarter_id == -1) {
2084                                 if (restarter_fmri != NULL)
2085                                         startd_free(restarter_fmri,
2086                                             max_scf_fmri_size);
2087                                 return (0);
2088                         }
2089 
2090                         graph_unset_restarter(v);
2091                 }
2092 
2093                 /* Master restarter, nothing to do. */
2094                 v->gv_restarter_id = -1;
2095                 v->gv_restarter_channel = NULL;
2096                 vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2097                 return (0);
2098         }
2099 
2100         if (v->gv_flags & GV_CONFIGURED) {
2101                 id = dict_lookup_byname(restarter_fmri);
2102                 if (id != -1 && v->gv_restarter_id == id) {
2103                         startd_free(restarter_fmri, max_scf_fmri_size);
2104                         return (0);
2105                 }
2106 
2107                 graph_unset_restarter(v);
2108         }
2109 
2110         err = graph_insert_vertex_unconfigured(restarter_fmri, GVT_INST, 0,
2111             RERR_NONE, &rv);
2112         startd_free(restarter_fmri, max_scf_fmri_size);
2113         assert(err == 0 || err == EEXIST);
2114 
2115         if (rv->gv_delegate_initialized == 0) {
2116                 if ((rv->gv_delegate_channel = restarter_protocol_init_delegate(
2117                     rv->gv_name)) == NULL)
2118                         return (EINVAL);
2119                 rv->gv_delegate_initialized = 1;
2120         }
2121         v->gv_restarter_id = rv->gv_id;
2122         v->gv_restarter_channel = rv->gv_delegate_channel;
2123 
2124         err = graph_insert_dependency(v, rv, pathp);
2125         if (err != 0) {
2126                 assert(err == ELOOP);
2127                 return (ELOOP);
2128         }
2129 
2130         vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2131 
2132         if (!(rv->gv_flags & GV_CONFIGURED)) {
2133                 scf_instance_t *inst;
2134 
2135                 err = libscf_fmri_get_instance(h, rv->gv_name, &inst);
2136                 switch (err) {
2137                 case 0:
2138                         err = configure_vertex(rv, inst);
2139                         scf_instance_destroy(inst);
2140                         switch (err) {
2141                         case 0:
2142                         case ECANCELED:
2143                                 break;
2144 
2145                         case ECONNABORTED:
2146                                 return (ECONNABORTED);
2147 
2148                         default:
2149                                 bad_error("configure_vertex", err);
2150                         }
2151                         break;
2152 
2153                 case ECONNABORTED:
2154                         return (ECONNABORTED);
2155 
2156                 case ENOENT:
2157                         break;
2158 
2159                 case ENOTSUP:
2160                         /*
2161                          * The fmri doesn't specify an instance - translate
2162                          * to EINVAL.
2163                          */
2164                         return (EINVAL);
2165 
2166                 case EINVAL:
2167                 default:
2168                         bad_error("libscf_fmri_get_instance", err);
2169                 }
2170         }
2171 
2172         return (0);
2173 }
2174 
2175 
2176 /*
2177  * Add all of the instances of the service named by fmri to the graph.
2178  * Returns
2179  *   0 - success
2180  *   ENOENT - service indicated by fmri does not exist
2181  *
2182  * In both cases *reboundp will be B_TRUE if the handle was rebound, or B_FALSE
2183  * otherwise.
2184  */
2185 static int
2186 add_service(const char *fmri, scf_handle_t *h, boolean_t *reboundp)
2187 {
2188         scf_service_t *svc;
2189         scf_instance_t *inst;
2190         scf_iter_t *iter;
2191         char *inst_fmri;
2192         int ret, r;
2193 
2194         *reboundp = B_FALSE;
2195 
2196         svc = safe_scf_service_create(h);
2197         inst = safe_scf_instance_create(h);
2198         iter = safe_scf_iter_create(h);
2199         inst_fmri = startd_alloc(max_scf_fmri_size);
2200 
2201 rebound:
2202         if (scf_handle_decode_fmri(h, fmri, NULL, svc, NULL, NULL, NULL,
2203             SCF_DECODE_FMRI_EXACT) != 0) {
2204                 switch (scf_error()) {
2205                 case SCF_ERROR_CONNECTION_BROKEN:
2206                 default:
2207                         libscf_handle_rebind(h);
2208                         *reboundp = B_TRUE;
2209                         goto rebound;
2210 
2211                 case SCF_ERROR_NOT_FOUND:
2212                         ret = ENOENT;
2213                         goto out;
2214 
2215                 case SCF_ERROR_INVALID_ARGUMENT:
2216                 case SCF_ERROR_CONSTRAINT_VIOLATED:
2217                 case SCF_ERROR_NOT_BOUND:
2218                 case SCF_ERROR_HANDLE_MISMATCH:
2219                         bad_error("scf_handle_decode_fmri", scf_error());
2220                 }
2221         }
2222 
2223         if (scf_iter_service_instances(iter, svc) != 0) {
2224                 switch (scf_error()) {
2225                 case SCF_ERROR_CONNECTION_BROKEN:
2226                 default:
2227                         libscf_handle_rebind(h);
2228                         *reboundp = B_TRUE;
2229                         goto rebound;
2230 
2231                 case SCF_ERROR_DELETED:
2232                         ret = ENOENT;
2233                         goto out;
2234 
2235                 case SCF_ERROR_HANDLE_MISMATCH:
2236                 case SCF_ERROR_NOT_BOUND:
2237                 case SCF_ERROR_NOT_SET:
2238                         bad_error("scf_iter_service_instances", scf_error());
2239                 }
2240         }
2241 
2242         for (;;) {
2243                 r = scf_iter_next_instance(iter, inst);
2244                 if (r == 0)
2245                         break;
2246                 if (r != 1) {
2247                         switch (scf_error()) {
2248                         case SCF_ERROR_CONNECTION_BROKEN:
2249                         default:
2250                                 libscf_handle_rebind(h);
2251                                 *reboundp = B_TRUE;
2252                                 goto rebound;
2253 
2254                         case SCF_ERROR_DELETED:
2255                                 ret = ENOENT;
2256                                 goto out;
2257 
2258                         case SCF_ERROR_HANDLE_MISMATCH:
2259                         case SCF_ERROR_NOT_BOUND:
2260                         case SCF_ERROR_NOT_SET:
2261                         case SCF_ERROR_INVALID_ARGUMENT:
2262                                 bad_error("scf_iter_next_instance",
2263                                     scf_error());
2264                         }
2265                 }
2266 
2267                 if (scf_instance_to_fmri(inst, inst_fmri, max_scf_fmri_size) <
2268                     0) {
2269                         switch (scf_error()) {
2270                         case SCF_ERROR_CONNECTION_BROKEN:
2271                                 libscf_handle_rebind(h);
2272                                 *reboundp = B_TRUE;
2273                                 goto rebound;
2274 
2275                         case SCF_ERROR_DELETED:
2276                                 continue;
2277 
2278                         case SCF_ERROR_NOT_BOUND:
2279                         case SCF_ERROR_NOT_SET:
2280                                 bad_error("scf_instance_to_fmri", scf_error());
2281                         }
2282                 }
2283 
2284                 r = dgraph_add_instance(inst_fmri, inst, B_FALSE);
2285                 switch (r) {
2286                 case 0:
2287                 case ECANCELED:
2288                         break;
2289 
2290                 case EEXIST:
2291                         continue;
2292 
2293                 case ECONNABORTED:
2294                         libscf_handle_rebind(h);
2295                         *reboundp = B_TRUE;
2296                         goto rebound;
2297 
2298                 case EINVAL:
2299                 default:
2300                         bad_error("dgraph_add_instance", r);
2301                 }
2302         }
2303 
2304         ret = 0;
2305 
2306 out:
2307         startd_free(inst_fmri, max_scf_fmri_size);
2308         scf_iter_destroy(iter);
2309         scf_instance_destroy(inst);
2310         scf_service_destroy(svc);
2311         return (ret);
2312 }
2313 
2314 struct depfmri_info {
2315         graph_vertex_t  *v;             /* GVT_GROUP vertex */
2316         gv_type_t       type;           /* type of dependency */
2317         const char      *inst_fmri;     /* FMRI of parental GVT_INST vert. */
2318         const char      *pg_name;       /* Name of dependency pg */
2319         scf_handle_t    *h;
2320         int             err;            /* return error code */
2321         int             **pathp;        /* return circular dependency path */
2322 };
2323 
2324 /*
2325  * Find or create a vertex for fmri and make info->v depend on it.
2326  * Returns
2327  *   0 - success
2328  *   nonzero - failure
2329  *
2330  * On failure, sets info->err to
2331  *   EINVAL - fmri is invalid
2332  *            fmri does not match info->type
2333  *   ELOOP - Adding the dependency creates a circular dependency.  *info->pathp
2334  *           will point to an array of the ids of the members of the cycle.
2335  *   ECONNABORTED - repository connection was broken
2336  *   ECONNRESET - succeeded, but repository connection was reset
2337  */
2338 static int
2339 process_dependency_fmri(const char *fmri, struct depfmri_info *info)
2340 {
2341         int err;
2342         graph_vertex_t *depgroup_v, *v;
2343         char *fmri_copy, *cfmri;
2344         size_t fmri_copy_sz;
2345         const char *scope, *service, *instance, *pg;
2346         scf_instance_t *inst;
2347         boolean_t rebound;
2348 
2349         assert(MUTEX_HELD(&dgraph_lock));
2350 
2351         /* Get or create vertex for FMRI */
2352         depgroup_v = info->v;
2353 
2354         if (strncmp(fmri, "file:", sizeof ("file:") - 1) == 0) {
2355                 if (info->type != GVT_FILE) {
2356                         log_framework(LOG_NOTICE,
2357                             "FMRI \"%s\" is not allowed for the \"%s\" "
2358                             "dependency's type of instance %s.\n", fmri,
2359                             info->pg_name, info->inst_fmri);
2360                         return (info->err = EINVAL);
2361                 }
2362 
2363                 err = graph_insert_vertex_unconfigured(fmri, info->type, 0,
2364                     RERR_NONE, &v);
2365                 switch (err) {
2366                 case 0:
2367                         break;
2368 
2369                 case EEXIST:
2370                         assert(v->gv_type == GVT_FILE);
2371                         break;
2372 
2373                 case EINVAL:            /* prevented above */
2374                 default:
2375                         bad_error("graph_insert_vertex_unconfigured", err);
2376                 }
2377         } else {
2378                 if (info->type != GVT_INST) {
2379                         log_framework(LOG_NOTICE,
2380                             "FMRI \"%s\" is not allowed for the \"%s\" "
2381                             "dependency's type of instance %s.\n", fmri,
2382                             info->pg_name, info->inst_fmri);
2383                         return (info->err = EINVAL);
2384                 }
2385 
2386                 /*
2387                  * We must canonify fmri & add a vertex for it.
2388                  */
2389                 fmri_copy_sz = strlen(fmri) + 1;
2390                 fmri_copy = startd_alloc(fmri_copy_sz);
2391                 (void) strcpy(fmri_copy, fmri);
2392 
2393                 /* Determine if the FMRI is a property group or instance */
2394                 if (scf_parse_svc_fmri(fmri_copy, &scope, &service,
2395                     &instance, &pg, NULL) != 0) {
2396                         startd_free(fmri_copy, fmri_copy_sz);
2397                         log_framework(LOG_NOTICE,
2398                             "Dependency \"%s\" of %s has invalid FMRI "
2399                             "\"%s\".\n", info->pg_name, info->inst_fmri,
2400                             fmri);
2401                         return (info->err = EINVAL);
2402                 }
2403 
2404                 if (service == NULL || pg != NULL) {
2405                         startd_free(fmri_copy, fmri_copy_sz);
2406                         log_framework(LOG_NOTICE,
2407                             "Dependency \"%s\" of %s does not designate a "
2408                             "service or instance.\n", info->pg_name,
2409                             info->inst_fmri);
2410                         return (info->err = EINVAL);
2411                 }
2412 
2413                 if (scope == NULL || strcmp(scope, SCF_SCOPE_LOCAL) == 0) {
2414                         cfmri = uu_msprintf("svc:/%s%s%s",
2415                             service, instance ? ":" : "", instance ? instance :
2416                             "");
2417                 } else {
2418                         cfmri = uu_msprintf("svc://%s/%s%s%s",
2419                             scope, service, instance ? ":" : "", instance ?
2420                             instance : "");
2421                 }
2422 
2423                 startd_free(fmri_copy, fmri_copy_sz);
2424 
2425                 err = graph_insert_vertex_unconfigured(cfmri, instance ?
2426                     GVT_INST : GVT_SVC, instance ? 0 : DEPGRP_REQUIRE_ANY,
2427                     RERR_NONE, &v);
2428                 uu_free(cfmri);
2429                 switch (err) {
2430                 case 0:
2431                         break;
2432 
2433                 case EEXIST:
2434                         /* Verify v. */
2435                         if (instance != NULL)
2436                                 assert(v->gv_type == GVT_INST);
2437                         else
2438                                 assert(v->gv_type == GVT_SVC);
2439                         break;
2440 
2441                 default:
2442                         bad_error("graph_insert_vertex_unconfigured", err);
2443                 }
2444         }
2445 
2446         /* Add dependency from depgroup_v to new vertex */
2447         info->err = graph_insert_dependency(depgroup_v, v, info->pathp);
2448         switch (info->err) {
2449         case 0:
2450                 break;
2451 
2452         case ELOOP:
2453                 return (ELOOP);
2454 
2455         default:
2456                 bad_error("graph_insert_dependency", info->err);
2457         }
2458 
2459         /* This must be after we insert the dependency, to avoid looping. */
2460         switch (v->gv_type) {
2461         case GVT_INST:
2462                 if ((v->gv_flags & GV_CONFIGURED) != 0)
2463                         break;
2464 
2465                 inst = safe_scf_instance_create(info->h);
2466 
2467                 rebound = B_FALSE;
2468 
2469 rebound:
2470                 err = libscf_lookup_instance(v->gv_name, inst);
2471                 switch (err) {
2472                 case 0:
2473                         err = configure_vertex(v, inst);
2474                         switch (err) {
2475                         case 0:
2476                         case ECANCELED:
2477                                 break;
2478 
2479                         case ECONNABORTED:
2480                                 libscf_handle_rebind(info->h);
2481                                 rebound = B_TRUE;
2482                                 goto rebound;
2483 
2484                         default:
2485                                 bad_error("configure_vertex", err);
2486                         }
2487                         break;
2488 
2489                 case ENOENT:
2490                         break;
2491 
2492                 case ECONNABORTED:
2493                         libscf_handle_rebind(info->h);
2494                         rebound = B_TRUE;
2495                         goto rebound;
2496 
2497                 case EINVAL:
2498                 case ENOTSUP:
2499                 default:
2500                         bad_error("libscf_fmri_get_instance", err);
2501                 }
2502 
2503                 scf_instance_destroy(inst);
2504 
2505                 if (rebound)
2506                         return (info->err = ECONNRESET);
2507                 break;
2508 
2509         case GVT_SVC:
2510                 (void) add_service(v->gv_name, info->h, &rebound);
2511                 if (rebound)
2512                         return (info->err = ECONNRESET);
2513         }
2514 
2515         return (0);
2516 }
2517 
2518 struct deppg_info {
2519         graph_vertex_t  *v;             /* GVT_INST vertex */
2520         int             err;            /* return error */
2521         int             **pathp;        /* return circular dependency path */
2522 };
2523 
2524 /*
2525  * Make info->v depend on a new GVT_GROUP node for this property group,
2526  * and then call process_dependency_fmri() for the values of the entity
2527  * property.  Return 0 on success, or if something goes wrong return nonzero
2528  * and set info->err to ECONNABORTED, EINVAL, or the error code returned by
2529  * process_dependency_fmri().
2530  */
2531 static int
2532 process_dependency_pg(scf_propertygroup_t *pg, struct deppg_info *info)
2533 {
2534         scf_handle_t *h;
2535         depgroup_type_t deptype;
2536         restarter_error_t rerr;
2537         struct depfmri_info linfo;
2538         char *fmri, *pg_name;
2539         size_t fmri_sz;
2540         graph_vertex_t *depgrp;
2541         scf_property_t *prop;
2542         int err;
2543         int empty;
2544         scf_error_t scferr;
2545         ssize_t len;
2546 
2547         assert(MUTEX_HELD(&dgraph_lock));
2548 
2549         h = scf_pg_handle(pg);
2550 
2551         pg_name = startd_alloc(max_scf_name_size);
2552 
2553         len = scf_pg_get_name(pg, pg_name, max_scf_name_size);
2554         if (len < 0) {
2555                 startd_free(pg_name, max_scf_name_size);
2556                 switch (scf_error()) {
2557                 case SCF_ERROR_CONNECTION_BROKEN:
2558                 default:
2559                         return (info->err = ECONNABORTED);
2560 
2561                 case SCF_ERROR_DELETED:
2562                         return (info->err = 0);
2563 
2564                 case SCF_ERROR_NOT_SET:
2565                         bad_error("scf_pg_get_name", scf_error());
2566                 }
2567         }
2568 
2569         /*
2570          * Skip over empty dependency groups.  Since dependency property
2571          * groups are updated atomically, they are either empty or
2572          * fully populated.
2573          */
2574         empty = depgroup_empty(h, pg);
2575         if (empty < 0) {
2576                 log_error(LOG_INFO,
2577                     "Error reading dependency group \"%s\" of %s: %s\n",
2578                     pg_name, info->v->gv_name, scf_strerror(scf_error()));
2579                 startd_free(pg_name, max_scf_name_size);
2580                 return (info->err = EINVAL);
2581 
2582         } else if (empty == 1) {
2583                 log_framework(LOG_DEBUG,
2584                     "Ignoring empty dependency group \"%s\" of %s\n",
2585                     pg_name, info->v->gv_name);
2586                 startd_free(pg_name, max_scf_name_size);
2587                 return (info->err = 0);
2588         }
2589 
2590         fmri_sz = strlen(info->v->gv_name) + 1 + len + 1;
2591         fmri = startd_alloc(fmri_sz);
2592 
2593         (void) snprintf(fmri, fmri_sz, "%s>%s", info->v->gv_name,
2594             pg_name);
2595 
2596         /* Validate the pg before modifying the graph */
2597         deptype = depgroup_read_grouping(h, pg);
2598         if (deptype == DEPGRP_UNSUPPORTED) {
2599                 log_error(LOG_INFO,
2600                     "Dependency \"%s\" of %s has an unknown grouping value.\n",
2601                     pg_name, info->v->gv_name);
2602                 startd_free(fmri, fmri_sz);
2603                 startd_free(pg_name, max_scf_name_size);
2604                 return (info->err = EINVAL);
2605         }
2606 
2607         rerr = depgroup_read_restart(h, pg);
2608         if (rerr == RERR_UNSUPPORTED) {
2609                 log_error(LOG_INFO,
2610                     "Dependency \"%s\" of %s has an unknown restart_on value."
2611                     "\n", pg_name, info->v->gv_name);
2612                 startd_free(fmri, fmri_sz);
2613                 startd_free(pg_name, max_scf_name_size);
2614                 return (info->err = EINVAL);
2615         }
2616 
2617         prop = safe_scf_property_create(h);
2618 
2619         if (scf_pg_get_property(pg, SCF_PROPERTY_ENTITIES, prop) != 0) {
2620                 scferr = scf_error();
2621                 scf_property_destroy(prop);
2622                 if (scferr == SCF_ERROR_DELETED) {
2623                         startd_free(fmri, fmri_sz);
2624                         startd_free(pg_name, max_scf_name_size);
2625                         return (info->err = 0);
2626                 } else if (scferr != SCF_ERROR_NOT_FOUND) {
2627                         startd_free(fmri, fmri_sz);
2628                         startd_free(pg_name, max_scf_name_size);
2629                         return (info->err = ECONNABORTED);
2630                 }
2631 
2632                 log_error(LOG_INFO,
2633                     "Dependency \"%s\" of %s is missing a \"%s\" property.\n",
2634                     pg_name, info->v->gv_name, SCF_PROPERTY_ENTITIES);
2635 
2636                 startd_free(fmri, fmri_sz);
2637                 startd_free(pg_name, max_scf_name_size);
2638 
2639                 return (info->err = EINVAL);
2640         }
2641 
2642         /* Create depgroup vertex for pg */
2643         err = graph_insert_vertex_unconfigured(fmri, GVT_GROUP, deptype,
2644             rerr, &depgrp);
2645         assert(err == 0);
2646         startd_free(fmri, fmri_sz);
2647 
2648         /* Add dependency from inst vertex to new vertex */
2649         err = graph_insert_dependency(info->v, depgrp, info->pathp);
2650         /* ELOOP can't happen because this should be a new vertex */
2651         assert(err == 0);
2652 
2653         linfo.v = depgrp;
2654         linfo.type = depgroup_read_scheme(h, pg);
2655         linfo.inst_fmri = info->v->gv_name;
2656         linfo.pg_name = pg_name;
2657         linfo.h = h;
2658         linfo.err = 0;
2659         linfo.pathp = info->pathp;
2660         err = walk_property_astrings(prop, (callback_t)process_dependency_fmri,
2661             &linfo);
2662 
2663         scf_property_destroy(prop);
2664         startd_free(pg_name, max_scf_name_size);
2665 
2666         switch (err) {
2667         case 0:
2668         case EINTR:
2669                 return (info->err = linfo.err);
2670 
2671         case ECONNABORTED:
2672         case EINVAL:
2673                 return (info->err = err);
2674 
2675         case ECANCELED:
2676                 return (info->err = 0);
2677 
2678         case ECONNRESET:
2679                 return (info->err = ECONNABORTED);
2680 
2681         default:
2682                 bad_error("walk_property_astrings", err);
2683                 /* NOTREACHED */
2684         }
2685 }
2686 
2687 /*
2688  * Build the dependency info for v from the repository.  Returns 0 on success,
2689  * ECONNABORTED on repository disconnection, EINVAL if the repository
2690  * configuration is invalid, and ELOOP if a dependency would cause a cycle.
2691  * In the last case, *pathp will point to a -1-terminated array of ids which
2692  * constitute the rest of the dependency cycle.
2693  */
2694 static int
2695 set_dependencies(graph_vertex_t *v, scf_instance_t *inst, int **pathp)
2696 {
2697         struct deppg_info info;
2698         int err;
2699         uint_t old_configured;
2700 
2701         assert(MUTEX_HELD(&dgraph_lock));
2702 
2703         /*
2704          * Mark the vertex as configured during dependency insertion to avoid
2705          * dependency cycles (which can appear in the graph if one of the
2706          * vertices is an exclusion-group).
2707          */
2708         old_configured = v->gv_flags & GV_CONFIGURED;
2709         v->gv_flags |= GV_CONFIGURED;
2710 
2711         info.err = 0;
2712         info.v = v;
2713         info.pathp = pathp;
2714 
2715         err = walk_dependency_pgs(inst, (callback_t)process_dependency_pg,
2716             &info);
2717 
2718         if (!old_configured)
2719                 v->gv_flags &= ~GV_CONFIGURED;
2720 
2721         switch (err) {
2722         case 0:
2723         case EINTR:
2724                 return (info.err);
2725 
2726         case ECONNABORTED:
2727                 return (ECONNABORTED);
2728 
2729         case ECANCELED:
2730                 /* Should get delete event, so return 0. */
2731                 return (0);
2732 
2733         default:
2734                 bad_error("walk_dependency_pgs", err);
2735                 /* NOTREACHED */
2736         }
2737 }
2738 
2739 
2740 static void
2741 handle_cycle(const char *fmri, int *path)
2742 {
2743         const char *cp;
2744         size_t sz;
2745 
2746         assert(MUTEX_HELD(&dgraph_lock));
2747 
2748         path_to_str(path, (char **)&cp, &sz);
2749 
2750         log_error(LOG_ERR, "Transitioning %s to maintenance "
2751             "because it completes a dependency cycle (see svcs -xv for "
2752             "details):\n%s", fmri ? fmri : "?", cp);
2753 
2754         startd_free((void *)cp, sz);
2755 }
2756 
2757 /*
2758  * Increment the vertex's reference count to prevent the vertex removal
2759  * from the dgraph.
2760  */
2761 static void
2762 vertex_ref(graph_vertex_t *v)
2763 {
2764         assert(MUTEX_HELD(&dgraph_lock));
2765 
2766         v->gv_refs++;
2767 }
2768 
2769 /*
2770  * Decrement the vertex's reference count and remove the vertex from
2771  * the dgraph when possible.
2772  *
2773  * Return VERTEX_REMOVED when the vertex has been removed otherwise
2774  * return VERTEX_INUSE.
2775  */
2776 static int
2777 vertex_unref(graph_vertex_t *v)
2778 {
2779         assert(MUTEX_HELD(&dgraph_lock));
2780         assert(v->gv_refs > 0);
2781 
2782         v->gv_refs--;
2783 
2784         return (free_if_unrefed(v));
2785 }
2786 
2787 /*
2788  * When run on the dependencies of a vertex, populates list with
2789  * graph_edge_t's which point to the service vertices or the instance
2790  * vertices (no GVT_GROUP nodes) on which the vertex depends.
2791  *
2792  * Increment the vertex's reference count once the vertex is inserted
2793  * in the list. The vertex won't be able to be deleted from the dgraph
2794  * while it is referenced.
2795  */
2796 static int
2797 append_svcs_or_insts(graph_edge_t *e, uu_list_t *list)
2798 {
2799         graph_vertex_t *v = e->ge_vertex;
2800         graph_edge_t *new;
2801         int r;
2802 
2803         switch (v->gv_type) {
2804         case GVT_INST:
2805         case GVT_SVC:
2806                 break;
2807 
2808         case GVT_GROUP:
2809                 r = uu_list_walk(v->gv_dependencies,
2810                     (uu_walk_fn_t *)append_svcs_or_insts, list, 0);
2811                 assert(r == 0);
2812                 return (UU_WALK_NEXT);
2813 
2814         case GVT_FILE:
2815                 return (UU_WALK_NEXT);
2816 
2817         default:
2818 #ifndef NDEBUG
2819                 uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
2820                     __LINE__, v->gv_type);
2821 #endif
2822                 abort();
2823         }
2824 
2825         new = startd_alloc(sizeof (*new));
2826         new->ge_vertex = v;
2827         uu_list_node_init(new, &new->ge_link, graph_edge_pool);
2828         r = uu_list_insert_before(list, NULL, new);
2829         assert(r == 0);
2830 
2831         /*
2832          * Because we are inserting the vertex in a list, we don't want
2833          * the vertex to be freed while the list is in use. In order to
2834          * achieve that, increment the vertex's reference count.
2835          */
2836         vertex_ref(v);
2837 
2838         return (UU_WALK_NEXT);
2839 }
2840 
2841 static boolean_t
2842 should_be_in_subgraph(graph_vertex_t *v)
2843 {
2844         graph_edge_t *e;
2845 
2846         if (v == milestone)
2847                 return (B_TRUE);
2848 
2849         /*
2850          * v is in the subgraph if any of its dependents are in the subgraph.
2851          * Except for EXCLUDE_ALL dependents.  And OPTIONAL dependents only
2852          * count if we're enabled.
2853          */
2854         for (e = uu_list_first(v->gv_dependents);
2855             e != NULL;
2856             e = uu_list_next(v->gv_dependents, e)) {
2857                 graph_vertex_t *dv = e->ge_vertex;
2858 
2859                 if (!(dv->gv_flags & GV_INSUBGRAPH))
2860                         continue;
2861 
2862                 /*
2863                  * Don't include instances that are optional and disabled.
2864                  */
2865                 if (v->gv_type == GVT_INST && dv->gv_type == GVT_SVC) {
2866 
2867                         int in = 0;
2868                         graph_edge_t *ee;
2869 
2870                         for (ee = uu_list_first(dv->gv_dependents);
2871                             ee != NULL;
2872                             ee = uu_list_next(dv->gv_dependents, ee)) {
2873 
2874                                 graph_vertex_t *ddv = e->ge_vertex;
2875 
2876                                 if (ddv->gv_type == GVT_GROUP &&
2877                                     ddv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2878                                         continue;
2879 
2880                                 if (ddv->gv_type == GVT_GROUP &&
2881                                     ddv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2882                                     !(v->gv_flags & GV_ENBLD_NOOVR))
2883                                         continue;
2884 
2885                                 in = 1;
2886                         }
2887                         if (!in)
2888                                 continue;
2889                 }
2890                 if (v->gv_type == GVT_INST &&
2891                     dv->gv_type == GVT_GROUP &&
2892                     dv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2893                     !(v->gv_flags & GV_ENBLD_NOOVR))
2894                         continue;
2895 
2896                 /* Don't include excluded services and instances */
2897                 if (dv->gv_type == GVT_GROUP &&
2898                     dv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2899                         continue;
2900 
2901                 return (B_TRUE);
2902         }
2903 
2904         return (B_FALSE);
2905 }
2906 
2907 /*
2908  * Ensures that GV_INSUBGRAPH is set properly for v and its descendents.  If
2909  * any bits change, manipulate the repository appropriately.  Returns 0 or
2910  * ECONNABORTED.
2911  */
2912 static int
2913 eval_subgraph(graph_vertex_t *v, scf_handle_t *h)
2914 {
2915         boolean_t old = (v->gv_flags & GV_INSUBGRAPH) != 0;
2916         boolean_t new;
2917         graph_edge_t *e;
2918         scf_instance_t *inst;
2919         int ret = 0, r;
2920 
2921         assert(milestone != NULL && milestone != MILESTONE_NONE);
2922 
2923         new = should_be_in_subgraph(v);
2924 
2925         if (new == old)
2926                 return (0);
2927 
2928         log_framework(LOG_DEBUG, new ? "Adding %s to the subgraph.\n" :
2929             "Removing %s from the subgraph.\n", v->gv_name);
2930 
2931         v->gv_flags = (v->gv_flags & ~GV_INSUBGRAPH) |
2932             (new ? GV_INSUBGRAPH : 0);
2933 
2934         if (v->gv_type == GVT_INST && (v->gv_flags & GV_CONFIGURED)) {
2935                 int err;
2936 
2937 get_inst:
2938                 err = libscf_fmri_get_instance(h, v->gv_name, &inst);
2939                 if (err != 0) {
2940                         switch (err) {
2941                         case ECONNABORTED:
2942                                 libscf_handle_rebind(h);
2943                                 ret = ECONNABORTED;
2944                                 goto get_inst;
2945 
2946                         case ENOENT:
2947                                 break;
2948 
2949                         case EINVAL:
2950                         case ENOTSUP:
2951                         default:
2952                                 bad_error("libscf_fmri_get_instance", err);
2953                         }
2954                 } else {
2955                         const char *f;
2956 
2957                         if (new) {
2958                                 err = libscf_delete_enable_ovr(inst);
2959                                 f = "libscf_delete_enable_ovr";
2960                         } else {
2961                                 err = libscf_set_enable_ovr(inst, 0);
2962                                 f = "libscf_set_enable_ovr";
2963                         }
2964                         scf_instance_destroy(inst);
2965                         switch (err) {
2966                         case 0:
2967                         case ECANCELED:
2968                                 break;
2969 
2970                         case ECONNABORTED:
2971                                 libscf_handle_rebind(h);
2972                                 /*
2973                                  * We must continue so the graph is updated,
2974                                  * but we must return ECONNABORTED so any
2975                                  * libscf state held by any callers is reset.
2976                                  */
2977                                 ret = ECONNABORTED;
2978                                 goto get_inst;
2979 
2980                         case EROFS:
2981                         case EPERM:
2982                                 log_error(LOG_WARNING,
2983                                     "Could not set %s/%s for %s: %s.\n",
2984                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
2985                                     v->gv_name, strerror(err));
2986                                 break;
2987 
2988                         default:
2989                                 bad_error(f, err);
2990                         }
2991                 }
2992         }
2993 
2994         for (e = uu_list_first(v->gv_dependencies);
2995             e != NULL;
2996             e = uu_list_next(v->gv_dependencies, e)) {
2997                 r = eval_subgraph(e->ge_vertex, h);
2998                 if (r != 0) {
2999                         assert(r == ECONNABORTED);
3000                         ret = ECONNABORTED;
3001                 }
3002         }
3003 
3004         return (ret);
3005 }
3006 
3007 /*
3008  * Delete the (property group) dependencies of v & create new ones based on
3009  * inst.  If doing so would create a cycle, log a message and put the instance
3010  * into maintenance.  Update GV_INSUBGRAPH flags as necessary.  Returns 0 or
3011  * ECONNABORTED.
3012  */
3013 int
3014 refresh_vertex(graph_vertex_t *v, scf_instance_t *inst)
3015 {
3016         int err;
3017         int *path;
3018         char *fmri;
3019         int r;
3020         scf_handle_t *h = scf_instance_handle(inst);
3021         uu_list_t *old_deps;
3022         int ret = 0;
3023         graph_edge_t *e;
3024         graph_vertex_t *vv;
3025 
3026         assert(MUTEX_HELD(&dgraph_lock));
3027         assert(v->gv_type == GVT_INST);
3028 
3029         log_framework(LOG_DEBUG, "Graph engine: Refreshing %s.\n", v->gv_name);
3030 
3031         if (milestone > MILESTONE_NONE) {
3032                 /*
3033                  * In case some of v's dependencies are being deleted we must
3034                  * make a list of them now for GV_INSUBGRAPH-flag evaluation
3035                  * after the new dependencies are in place.
3036                  */
3037                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
3038 
3039                 err = uu_list_walk(v->gv_dependencies,
3040                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
3041                 assert(err == 0);
3042         }
3043 
3044         delete_instance_dependencies(v, B_FALSE);
3045 
3046         err = set_dependencies(v, inst, &path);
3047         switch (err) {
3048         case 0:
3049                 break;
3050 
3051         case ECONNABORTED:
3052                 ret = err;
3053                 goto out;
3054 
3055         case EINVAL:
3056         case ELOOP:
3057                 r = libscf_instance_get_fmri(inst, &fmri);
3058                 switch (r) {
3059                 case 0:
3060                         break;
3061 
3062                 case ECONNABORTED:
3063                         ret = ECONNABORTED;
3064                         goto out;
3065 
3066                 case ECANCELED:
3067                         ret = 0;
3068                         goto out;
3069 
3070                 default:
3071                         bad_error("libscf_instance_get_fmri", r);
3072                 }
3073 
3074                 if (err == EINVAL) {
3075                         log_error(LOG_ERR, "Transitioning %s "
3076                             "to maintenance due to misconfiguration.\n",
3077                             fmri ? fmri : "?");
3078                         vertex_send_event(v,
3079                             RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY);
3080                 } else {
3081                         handle_cycle(fmri, path);
3082                         vertex_send_event(v,
3083                             RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE);
3084                 }
3085                 startd_free(fmri, max_scf_fmri_size);
3086                 ret = 0;
3087                 goto out;
3088 
3089         default:
3090                 bad_error("set_dependencies", err);
3091         }
3092 
3093         if (milestone > MILESTONE_NONE) {
3094                 boolean_t aborted = B_FALSE;
3095 
3096                 for (e = uu_list_first(old_deps);
3097                     e != NULL;
3098                     e = uu_list_next(old_deps, e)) {
3099                         vv = e->ge_vertex;
3100 
3101                         if (vertex_unref(vv) == VERTEX_INUSE &&
3102                             eval_subgraph(vv, h) == ECONNABORTED)
3103                                 aborted = B_TRUE;
3104                 }
3105 
3106                 for (e = uu_list_first(v->gv_dependencies);
3107                     e != NULL;
3108                     e = uu_list_next(v->gv_dependencies, e)) {
3109                         if (eval_subgraph(e->ge_vertex, h) ==
3110                             ECONNABORTED)
3111                                 aborted = B_TRUE;
3112                 }
3113 
3114                 if (aborted) {
3115                         ret = ECONNABORTED;
3116                         goto out;
3117                 }
3118         }
3119 
3120         graph_start_if_satisfied(v);
3121 
3122         ret = 0;
3123 
3124 out:
3125         if (milestone > MILESTONE_NONE) {
3126                 void *cookie = NULL;
3127 
3128                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL)
3129                         startd_free(e, sizeof (*e));
3130 
3131                 uu_list_destroy(old_deps);
3132         }
3133 
3134         return (ret);
3135 }
3136 
3137 /*
3138  * Set up v according to inst.  That is, make sure it depends on its
3139  * restarter and set up its dependencies.  Send the ADD_INSTANCE command to
3140  * the restarter, and send ENABLE or DISABLE as appropriate.
3141  *
3142  * Returns 0 on success, ECONNABORTED on repository disconnection, or
3143  * ECANCELED if inst is deleted.
3144  */
3145 static int
3146 configure_vertex(graph_vertex_t *v, scf_instance_t *inst)
3147 {
3148         scf_handle_t *h;
3149         scf_propertygroup_t *pg;
3150         scf_snapshot_t *snap;
3151         char *restarter_fmri = startd_alloc(max_scf_value_size);
3152         int enabled, enabled_ovr;
3153         int err;
3154         int *path;
3155         int deathrow;
3156         int32_t tset;
3157 
3158         restarter_fmri[0] = '\0';
3159 
3160         assert(MUTEX_HELD(&dgraph_lock));
3161         assert(v->gv_type == GVT_INST);
3162         assert((v->gv_flags & GV_CONFIGURED) == 0);
3163 
3164         /* GV_INSUBGRAPH should already be set properly. */
3165         assert(should_be_in_subgraph(v) ==
3166             ((v->gv_flags & GV_INSUBGRAPH) != 0));
3167 
3168         /*
3169          * If the instance fmri is in the deathrow list then set the
3170          * GV_DEATHROW flag on the vertex and create and set to true the
3171          * SCF_PROPERTY_DEATHROW boolean property in the non-persistent
3172          * repository for this instance fmri.
3173          */
3174         if ((v->gv_flags & GV_DEATHROW) ||
3175             (is_fmri_in_deathrow(v->gv_name) == B_TRUE)) {
3176                 if ((v->gv_flags & GV_DEATHROW) == 0) {
3177                         /*
3178                          * Set flag GV_DEATHROW, create and set to true
3179                          * the SCF_PROPERTY_DEATHROW property in the
3180                          * non-persistent repository for this instance fmri.
3181                          */
3182                         v->gv_flags |= GV_DEATHROW;
3183 
3184                         switch (err = libscf_set_deathrow(inst, 1)) {
3185                         case 0:
3186                                 break;
3187 
3188                         case ECONNABORTED:
3189                         case ECANCELED:
3190                                 startd_free(restarter_fmri, max_scf_value_size);
3191                                 return (err);
3192 
3193                         case EROFS:
3194                                 log_error(LOG_WARNING, "Could not set %s/%s "
3195                                     "for deathrow %s: %s.\n",
3196                                     SCF_PG_DEATHROW, SCF_PROPERTY_DEATHROW,
3197                                     v->gv_name, strerror(err));
3198                                 break;
3199 
3200                         case EPERM:
3201                                 uu_die("Permission denied.\n");
3202                                 /* NOTREACHED */
3203 
3204                         default:
3205                                 bad_error("libscf_set_deathrow", err);
3206                         }
3207                         log_framework(LOG_DEBUG, "Deathrow, graph set %s.\n",
3208                             v->gv_name);
3209                 }
3210                 startd_free(restarter_fmri, max_scf_value_size);
3211                 return (0);
3212         }
3213 
3214         h = scf_instance_handle(inst);
3215 
3216         /*
3217          * Using a temporary deathrow boolean property, set through
3218          * libscf_set_deathrow(), only for fmris on deathrow, is necessary
3219          * because deathrow_fini() may already have been called, and in case
3220          * of a refresh, GV_DEATHROW may need to be set again.
3221          * libscf_get_deathrow() sets deathrow to 1 only if this instance
3222          * has a temporary boolean property named 'deathrow' valued true
3223          * in a property group 'deathrow', -1 or 0 in all other cases.
3224          */
3225         err = libscf_get_deathrow(h, inst, &deathrow);
3226         switch (err) {
3227         case 0:
3228                 break;
3229 
3230         case ECONNABORTED:
3231         case ECANCELED:
3232                 startd_free(restarter_fmri, max_scf_value_size);
3233                 return (err);
3234 
3235         default:
3236                 bad_error("libscf_get_deathrow", err);
3237         }
3238 
3239         if (deathrow == 1) {
3240                 v->gv_flags |= GV_DEATHROW;
3241                 startd_free(restarter_fmri, max_scf_value_size);
3242                 return (0);
3243         }
3244 
3245         log_framework(LOG_DEBUG, "Graph adding %s.\n", v->gv_name);
3246 
3247         /*
3248          * If the instance does not have a restarter property group,
3249          * initialize its state to uninitialized/none, in case the restarter
3250          * is not enabled.
3251          */
3252         pg = safe_scf_pg_create(h);
3253 
3254         if (scf_instance_get_pg(inst, SCF_PG_RESTARTER, pg) != 0) {
3255                 instance_data_t idata;
3256                 uint_t count = 0, msecs = ALLOC_DELAY;
3257 
3258                 switch (scf_error()) {
3259                 case SCF_ERROR_NOT_FOUND:
3260                         break;
3261 
3262                 case SCF_ERROR_CONNECTION_BROKEN:
3263                 default:
3264                         scf_pg_destroy(pg);
3265                         startd_free(restarter_fmri, max_scf_value_size);
3266                         return (ECONNABORTED);
3267 
3268                 case SCF_ERROR_DELETED:
3269                         scf_pg_destroy(pg);
3270                         startd_free(restarter_fmri, max_scf_value_size);
3271                         return (ECANCELED);
3272 
3273                 case SCF_ERROR_NOT_SET:
3274                         bad_error("scf_instance_get_pg", scf_error());
3275                 }
3276 
3277                 switch (err = libscf_instance_get_fmri(inst,
3278                     (char **)&idata.i_fmri)) {
3279                 case 0:
3280                         break;
3281 
3282                 case ECONNABORTED:
3283                 case ECANCELED:
3284                         scf_pg_destroy(pg);
3285                         startd_free(restarter_fmri, max_scf_value_size);
3286                         return (err);
3287 
3288                 default:
3289                         bad_error("libscf_instance_get_fmri", err);
3290                 }
3291 
3292                 idata.i_state = RESTARTER_STATE_NONE;
3293                 idata.i_next_state = RESTARTER_STATE_NONE;
3294 
3295 init_state:
3296                 switch (err = _restarter_commit_states(h, &idata,
3297                     RESTARTER_STATE_UNINIT, RESTARTER_STATE_NONE,
3298                     restarter_get_str_short(restarter_str_insert_in_graph))) {
3299                 case 0:
3300                         break;
3301 
3302                 case ENOMEM:
3303                         ++count;
3304                         if (count < ALLOC_RETRY) {
3305                                 (void) poll(NULL, 0, msecs);
3306                                 msecs *= ALLOC_DELAY_MULT;
3307                                 goto init_state;
3308                         }
3309 
3310                         uu_die("Insufficient memory.\n");
3311                         /* NOTREACHED */
3312 
3313                 case ECONNABORTED:
3314                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3315                         scf_pg_destroy(pg);
3316                         startd_free(restarter_fmri, max_scf_value_size);
3317                         return (ECONNABORTED);
3318 
3319                 case ENOENT:
3320                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3321                         scf_pg_destroy(pg);
3322                         startd_free(restarter_fmri, max_scf_value_size);
3323                         return (ECANCELED);
3324 
3325                 case EPERM:
3326                 case EACCES:
3327                 case EROFS:
3328                         log_error(LOG_NOTICE, "Could not initialize state for "
3329                             "%s: %s.\n", idata.i_fmri, strerror(err));
3330                         break;
3331 
3332                 case EINVAL:
3333                 default:
3334                         bad_error("_restarter_commit_states", err);
3335                 }
3336 
3337                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3338         }
3339 
3340         scf_pg_destroy(pg);
3341 
3342         if (milestone != NULL) {
3343                 /*
3344                  * Make sure the enable-override is set properly before we
3345                  * read whether we should be enabled.
3346                  */
3347                 if (milestone == MILESTONE_NONE ||
3348                     !(v->gv_flags & GV_INSUBGRAPH)) {
3349                         /*
3350                          * This might seem unjustified after the milestone
3351                          * transition has completed (non_subgraph_svcs == 0),
3352                          * but it's important because when we boot to
3353                          * a milestone, we set the milestone before populating
3354                          * the graph, and all of the new non-subgraph services
3355                          * need to be disabled here.
3356                          */
3357                         switch (err = libscf_set_enable_ovr(inst, 0)) {
3358                         case 0:
3359                                 break;
3360 
3361                         case ECONNABORTED:
3362                         case ECANCELED:
3363                                 startd_free(restarter_fmri, max_scf_value_size);
3364                                 return (err);
3365 
3366                         case EROFS:
3367                                 log_error(LOG_WARNING,
3368                                     "Could not set %s/%s for %s: %s.\n",
3369                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
3370                                     v->gv_name, strerror(err));
3371                                 break;
3372 
3373                         case EPERM:
3374                                 uu_die("Permission denied.\n");
3375                                 /* NOTREACHED */
3376 
3377                         default:
3378                                 bad_error("libscf_set_enable_ovr", err);
3379                         }
3380                 } else {
3381                         assert(v->gv_flags & GV_INSUBGRAPH);
3382                         switch (err = libscf_delete_enable_ovr(inst)) {
3383                         case 0:
3384                                 break;
3385 
3386                         case ECONNABORTED:
3387                         case ECANCELED:
3388                                 startd_free(restarter_fmri, max_scf_value_size);
3389                                 return (err);
3390 
3391                         case EPERM:
3392                                 uu_die("Permission denied.\n");
3393                                 /* NOTREACHED */
3394 
3395                         default:
3396                                 bad_error("libscf_delete_enable_ovr", err);
3397                         }
3398                 }
3399         }
3400 
3401         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
3402             &enabled_ovr, &restarter_fmri);
3403         switch (err) {
3404         case 0:
3405                 break;
3406 
3407         case ECONNABORTED:
3408         case ECANCELED:
3409                 startd_free(restarter_fmri, max_scf_value_size);
3410                 return (err);
3411 
3412         case ENOENT:
3413                 log_framework(LOG_DEBUG,
3414                     "Ignoring %s because it has no general property group.\n",
3415                     v->gv_name);
3416                 startd_free(restarter_fmri, max_scf_value_size);
3417                 return (0);
3418 
3419         default:
3420                 bad_error("libscf_get_basic_instance_data", err);
3421         }
3422 
3423         if ((tset = libscf_get_stn_tset(inst)) == -1) {
3424                 log_framework(LOG_WARNING,
3425                     "Failed to get notification parameters for %s: %s\n",
3426                     v->gv_name, scf_strerror(scf_error()));
3427                 v->gv_stn_tset = 0;
3428         } else {
3429                 v->gv_stn_tset = tset;
3430         }
3431         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
3432                 stn_global = v->gv_stn_tset;
3433 
3434         if (enabled == -1) {
3435                 startd_free(restarter_fmri, max_scf_value_size);
3436                 return (0);
3437         }
3438 
3439         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
3440             (enabled ? GV_ENBLD_NOOVR : 0);
3441 
3442         if (enabled_ovr != -1)
3443                 enabled = enabled_ovr;
3444 
3445         v->gv_state = RESTARTER_STATE_UNINIT;
3446 
3447         snap = libscf_get_or_make_running_snapshot(inst, v->gv_name, B_TRUE);
3448         scf_snapshot_destroy(snap);
3449 
3450         /* Set up the restarter. (Sends _ADD_INSTANCE on success.) */
3451         err = graph_change_restarter(v, restarter_fmri, h, &path);
3452         if (err != 0) {
3453                 instance_data_t idata;
3454                 uint_t count = 0, msecs = ALLOC_DELAY;
3455                 restarter_str_t reason;
3456 
3457                 if (err == ECONNABORTED) {
3458                         startd_free(restarter_fmri, max_scf_value_size);
3459                         return (err);
3460                 }
3461 
3462                 assert(err == EINVAL || err == ELOOP);
3463 
3464                 if (err == EINVAL) {
3465                         log_framework(LOG_ERR, emsg_invalid_restarter,
3466                             v->gv_name, restarter_fmri);
3467                         reason = restarter_str_invalid_restarter;
3468                 } else {
3469                         handle_cycle(v->gv_name, path);
3470                         reason = restarter_str_dependency_cycle;
3471                 }
3472 
3473                 startd_free(restarter_fmri, max_scf_value_size);
3474 
3475                 /*
3476                  * We didn't register the instance with the restarter, so we
3477                  * must set maintenance mode ourselves.
3478                  */
3479                 err = libscf_instance_get_fmri(inst, (char **)&idata.i_fmri);
3480                 if (err != 0) {
3481                         assert(err == ECONNABORTED || err == ECANCELED);
3482                         return (err);
3483                 }
3484 
3485                 idata.i_state = RESTARTER_STATE_NONE;
3486                 idata.i_next_state = RESTARTER_STATE_NONE;
3487 
3488 set_maint:
3489                 switch (err = _restarter_commit_states(h, &idata,
3490                     RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE,
3491                     restarter_get_str_short(reason))) {
3492                 case 0:
3493                         break;
3494 
3495                 case ENOMEM:
3496                         ++count;
3497                         if (count < ALLOC_RETRY) {
3498                                 (void) poll(NULL, 0, msecs);
3499                                 msecs *= ALLOC_DELAY_MULT;
3500                                 goto set_maint;
3501                         }
3502 
3503                         uu_die("Insufficient memory.\n");
3504                         /* NOTREACHED */
3505 
3506                 case ECONNABORTED:
3507                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3508                         return (ECONNABORTED);
3509 
3510                 case ENOENT:
3511                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3512                         return (ECANCELED);
3513 
3514                 case EPERM:
3515                 case EACCES:
3516                 case EROFS:
3517                         log_error(LOG_NOTICE, "Could not initialize state for "
3518                             "%s: %s.\n", idata.i_fmri, strerror(err));
3519                         break;
3520 
3521                 case EINVAL:
3522                 default:
3523                         bad_error("_restarter_commit_states", err);
3524                 }
3525 
3526                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3527 
3528                 v->gv_state = RESTARTER_STATE_MAINT;
3529 
3530                 goto out;
3531         }
3532         startd_free(restarter_fmri, max_scf_value_size);
3533 
3534         /* Add all the other dependencies. */
3535         err = refresh_vertex(v, inst);
3536         if (err != 0) {
3537                 assert(err == ECONNABORTED);
3538                 return (err);
3539         }
3540 
3541 out:
3542         v->gv_flags |= GV_CONFIGURED;
3543 
3544         graph_enable_by_vertex(v, enabled, 0);
3545 
3546         return (0);
3547 }
3548 
3549 
3550 static void
3551 kill_user_procs(void)
3552 {
3553         (void) fputs("svc.startd: Killing user processes.\n", stdout);
3554 
3555         /*
3556          * Despite its name, killall's role is to get select user processes--
3557          * basically those representing terminal-based logins-- to die.  Victims
3558          * are located by killall in the utmp database.  Since these are most
3559          * often shell based logins, and many shells mask SIGTERM (but are
3560          * responsive to SIGHUP) we first HUP and then shortly thereafter
3561          * kill -9.
3562          */
3563         (void) fork_with_timeout("/usr/sbin/killall HUP", 1, 5);
3564         (void) fork_with_timeout("/usr/sbin/killall KILL", 1, 5);
3565 
3566         /*
3567          * Note the selection of user id's 0, 1 and 15, subsequently
3568          * inverted by -v.  15 is reserved for dladmd.  Yes, this is a
3569          * kludge-- a better policy is needed.
3570          *
3571          * Note that fork_with_timeout will only wait out the 1 second
3572          * "grace time" if pkill actually returns 0.  So if there are
3573          * no matches, this will run to completion much more quickly.
3574          */
3575         (void) fork_with_timeout("/usr/bin/pkill -TERM -v -u 0,1,15", 1, 5);
3576         (void) fork_with_timeout("/usr/bin/pkill -KILL -v -u 0,1,15", 1, 5);
3577 }
3578 
3579 static void
3580 do_uadmin(void)
3581 {
3582         const char * const resetting = "/etc/svc/volatile/resetting";
3583         int fd;
3584         struct statvfs vfs;
3585         time_t now;
3586         struct tm nowtm;
3587         char down_buf[256], time_buf[256];
3588         uintptr_t mdep;
3589 #if defined(__x86)
3590         char *fbarg = NULL;
3591 #endif  /* __x86 */
3592 
3593         mdep = NULL;
3594         fd = creat(resetting, 0777);
3595         if (fd >= 0)
3596                 startd_close(fd);
3597         else
3598                 uu_warn("Could not create \"%s\"", resetting);
3599 
3600         /* Kill dhcpagent if we're not using nfs for root */
3601         if ((statvfs("/", &vfs) == 0) &&
3602             (strncmp(vfs.f_basetype, "nfs", sizeof ("nfs") - 1) != 0))
3603                 fork_with_timeout("/usr/bin/pkill -x -u 0 dhcpagent", 0, 5);
3604 
3605         /*
3606          * Call sync(2) now, before we kill off user processes.  This takes
3607          * advantage of the several seconds of pause we have before the
3608          * killalls are done.  Time we can make good use of to get pages
3609          * moving out to disk.
3610          *
3611          * Inside non-global zones, we don't bother, and it's better not to
3612          * anyway, since sync(2) can have system-wide impact.
3613          */
3614         if (getzoneid() == 0)
3615                 sync();
3616 
3617         kill_user_procs();
3618 
3619         /*
3620          * Note that this must come after the killing of user procs, since
3621          * killall relies on utmpx, and this command affects the contents of
3622          * said file.
3623          */
3624         if (access("/usr/lib/acct/closewtmp", X_OK) == 0)
3625                 fork_with_timeout("/usr/lib/acct/closewtmp", 0, 5);
3626 
3627         /*
3628          * For patches which may be installed as the system is shutting
3629          * down, we need to ensure, one more time, that the boot archive
3630          * really is up to date.
3631          */
3632         if (getzoneid() == 0 && access("/usr/sbin/bootadm", X_OK) == 0)
3633                 fork_with_timeout("/usr/sbin/bootadm -ea update_all", 0, 3600);
3634 
3635         /*
3636          * Right now, fast reboot is supported only on i386.
3637          * scf_is_fastboot_default() should take care of it.
3638          * If somehow we got there on unsupported platform -
3639          * print warning and fall back to regular reboot.
3640          */
3641         if (halting == AD_FASTREBOOT) {
3642 #if defined(__x86)
3643                 if (be_get_boot_args(&fbarg, BE_ENTRY_DEFAULT) == 0) {
3644                         mdep = (uintptr_t)fbarg;
3645                 } else {
3646                         /*
3647                          * Failed to read BE info, fall back to normal reboot
3648                          */
3649                         halting = AD_BOOT;
3650                         uu_warn("Failed to get fast reboot arguments.\n"
3651                             "Falling back to regular reboot.\n");
3652                 }
3653 #else   /* __x86 */
3654                 halting = AD_BOOT;
3655                 uu_warn("Fast reboot configured, but not supported by "
3656                     "this ISA\n");
3657 #endif  /* __x86 */
3658         }
3659 
3660         fork_with_timeout("/sbin/umountall -l", 0, 5);
3661         fork_with_timeout("/sbin/umount /tmp /var/adm /var/run /var "
3662             ">/dev/null 2>&1", 0, 5);
3663 
3664         /*
3665          * Try to get to consistency for whatever UFS filesystems are left.
3666          * This is pretty expensive, so we save it for the end in the hopes of
3667          * minimizing what it must do.  The other option would be to start in
3668          * parallel with the killall's, but lockfs tends to throw out much more
3669          * than is needed, and so subsequent commands (like umountall) take a
3670          * long time to get going again.
3671          *
3672          * Inside of zones, we don't bother, since we're not about to terminate
3673          * the whole OS instance.
3674          *
3675          * On systems using only ZFS, this call to lockfs -fa is a no-op.
3676          */
3677         if (getzoneid() == 0) {
3678                 if (access("/usr/sbin/lockfs", X_OK) == 0)
3679                         fork_with_timeout("/usr/sbin/lockfs -fa", 0, 30);
3680 
3681                 sync(); /* once more, with feeling */
3682         }
3683 
3684         fork_with_timeout("/sbin/umount /usr >/dev/null 2>&1", 0, 5);
3685 
3686         /*
3687          * Construct and emit the last words from userland:
3688          * "<timestamp> The system is down.  Shutdown took <N> seconds."
3689          *
3690          * Normally we'd use syslog, but with /var and other things
3691          * potentially gone, try to minimize the external dependencies.
3692          */
3693         now = time(NULL);
3694         (void) localtime_r(&now, &nowtm);
3695 
3696         if (strftime(down_buf, sizeof (down_buf),
3697             "%b %e %T The system is down.", &nowtm) == 0) {
3698                 (void) strlcpy(down_buf, "The system is down.",
3699                     sizeof (down_buf));
3700         }
3701 
3702         if (halting_time != 0 && halting_time <= now) {
3703                 (void) snprintf(time_buf, sizeof (time_buf),
3704                     "  Shutdown took %lu seconds.", now - halting_time);
3705         } else {
3706                 time_buf[0] = '\0';
3707         }
3708         (void) printf("%s%s\n", down_buf, time_buf);
3709 
3710         (void) uadmin(A_SHUTDOWN, halting, mdep);
3711         uu_warn("uadmin() failed");
3712 
3713 #if defined(__x86)
3714         if (halting == AD_FASTREBOOT)
3715                 free(fbarg);
3716 #endif  /* __x86 */
3717 
3718         if (remove(resetting) != 0 && errno != ENOENT)
3719                 uu_warn("Could not remove \"%s\"", resetting);
3720 }
3721 
3722 /*
3723  * If any of the up_svcs[] are online or satisfiable, return true.  If they are
3724  * all missing, disabled, in maintenance, or unsatisfiable, return false.
3725  */
3726 boolean_t
3727 can_come_up(void)
3728 {
3729         int i;
3730 
3731         assert(MUTEX_HELD(&dgraph_lock));
3732 
3733         /*
3734          * If we are booting to single user (boot -s),
3735          * SCF_MILESTONE_SINGLE_USER is needed to come up because startd
3736          * spawns sulogin after single-user is online (see specials.c).
3737          */
3738         i = (booting_to_single_user ? 0 : 1);
3739 
3740         for (; up_svcs[i] != NULL; ++i) {
3741                 if (up_svcs_p[i] == NULL) {
3742                         up_svcs_p[i] = vertex_get_by_name(up_svcs[i]);
3743 
3744                         if (up_svcs_p[i] == NULL)
3745                                 continue;
3746                 }
3747 
3748                 /*
3749                  * Ignore unconfigured services (the ones that have been
3750                  * mentioned in a dependency from other services, but do
3751                  * not exist in the repository).  Services which exist
3752                  * in the repository but don't have general/enabled
3753                  * property will be also ignored.
3754                  */
3755                 if (!(up_svcs_p[i]->gv_flags & GV_CONFIGURED))
3756                         continue;
3757 
3758                 switch (up_svcs_p[i]->gv_state) {
3759                 case RESTARTER_STATE_ONLINE:
3760                 case RESTARTER_STATE_DEGRADED:
3761                         /*
3762                          * Deactivate verbose boot once a login service has been
3763                          * reached.
3764                          */
3765                         st->st_log_login_reached = 1;
3766                         /*FALLTHROUGH*/
3767                 case RESTARTER_STATE_UNINIT:
3768                         return (B_TRUE);
3769 
3770                 case RESTARTER_STATE_OFFLINE:
3771                         if (instance_satisfied(up_svcs_p[i], B_TRUE) != -1)
3772                                 return (B_TRUE);
3773                         log_framework(LOG_DEBUG,
3774                             "can_come_up(): %s is unsatisfiable.\n",
3775                             up_svcs_p[i]->gv_name);
3776                         continue;
3777 
3778                 case RESTARTER_STATE_DISABLED:
3779                 case RESTARTER_STATE_MAINT:
3780                         log_framework(LOG_DEBUG,
3781                             "can_come_up(): %s is in state %s.\n",
3782                             up_svcs_p[i]->gv_name,
3783                             instance_state_str[up_svcs_p[i]->gv_state]);
3784                         continue;
3785 
3786                 default:
3787 #ifndef NDEBUG
3788                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
3789                             __FILE__, __LINE__, up_svcs_p[i]->gv_state);
3790 #endif
3791                         abort();
3792                 }
3793         }
3794 
3795         /*
3796          * In the seed repository, console-login is unsatisfiable because
3797          * services are missing.  To behave correctly in that case we don't want
3798          * to return false until manifest-import is online.
3799          */
3800 
3801         if (manifest_import_p == NULL) {
3802                 manifest_import_p = vertex_get_by_name(manifest_import);
3803 
3804                 if (manifest_import_p == NULL)
3805                         return (B_FALSE);
3806         }
3807 
3808         switch (manifest_import_p->gv_state) {
3809         case RESTARTER_STATE_ONLINE:
3810         case RESTARTER_STATE_DEGRADED:
3811         case RESTARTER_STATE_DISABLED:
3812         case RESTARTER_STATE_MAINT:
3813                 break;
3814 
3815         case RESTARTER_STATE_OFFLINE:
3816                 if (instance_satisfied(manifest_import_p, B_TRUE) == -1)
3817                         break;
3818                 /* FALLTHROUGH */
3819 
3820         case RESTARTER_STATE_UNINIT:
3821                 return (B_TRUE);
3822         }
3823 
3824         return (B_FALSE);
3825 }
3826 
3827 /*
3828  * Runs sulogin.  Returns
3829  *   0 - success
3830  *   EALREADY - sulogin is already running
3831  *   EBUSY - console-login is running
3832  */
3833 static int
3834 run_sulogin(const char *msg)
3835 {
3836         graph_vertex_t *v;
3837 
3838         assert(MUTEX_HELD(&dgraph_lock));
3839 
3840         if (sulogin_running)
3841                 return (EALREADY);
3842 
3843         v = vertex_get_by_name(console_login_fmri);
3844         if (v != NULL && inst_running(v))
3845                 return (EBUSY);
3846 
3847         sulogin_running = B_TRUE;
3848 
3849         MUTEX_UNLOCK(&dgraph_lock);
3850 
3851         fork_sulogin(B_FALSE, msg);
3852 
3853         MUTEX_LOCK(&dgraph_lock);
3854 
3855         sulogin_running = B_FALSE;
3856 
3857         if (console_login_ready) {
3858                 v = vertex_get_by_name(console_login_fmri);
3859 
3860                 if (v != NULL && v->gv_state == RESTARTER_STATE_OFFLINE) {
3861                         if (v->gv_start_f == NULL)
3862                                 vertex_send_event(v,
3863                                     RESTARTER_EVENT_TYPE_START);
3864                         else
3865                                 v->gv_start_f(v);
3866                 }
3867 
3868                 console_login_ready = B_FALSE;
3869         }
3870 
3871         return (0);
3872 }
3873 
3874 /*
3875  * The sulogin thread runs sulogin while can_come_up() is false.  run_sulogin()
3876  * keeps sulogin from stepping on console-login's toes.
3877  */
3878 /* ARGSUSED */
3879 static void *
3880 sulogin_thread(void *unused)
3881 {
3882         MUTEX_LOCK(&dgraph_lock);
3883 
3884         assert(sulogin_thread_running);
3885 
3886         do {
3887                 (void) run_sulogin("Console login service(s) cannot run\n");
3888         } while (!can_come_up());
3889 
3890         sulogin_thread_running = B_FALSE;
3891         MUTEX_UNLOCK(&dgraph_lock);
3892 
3893         return (NULL);
3894 }
3895 
3896 /* ARGSUSED */
3897 void *
3898 single_user_thread(void *unused)
3899 {
3900         uint_t left;
3901         scf_handle_t *h;
3902         scf_instance_t *inst;
3903         scf_property_t *prop;
3904         scf_value_t *val;
3905         const char *msg;
3906         char *buf;
3907         int r;
3908 
3909         MUTEX_LOCK(&single_user_thread_lock);
3910         single_user_thread_count++;
3911 
3912         if (!booting_to_single_user)
3913                 kill_user_procs();
3914 
3915         if (go_single_user_mode || booting_to_single_user) {
3916                 msg = "SINGLE USER MODE\n";
3917         } else {
3918                 assert(go_to_level1);
3919 
3920                 fork_rc_script('1', "start", B_TRUE);
3921 
3922                 uu_warn("The system is ready for administration.\n");
3923 
3924                 msg = "";
3925         }
3926 
3927         MUTEX_UNLOCK(&single_user_thread_lock);
3928 
3929         for (;;) {
3930                 MUTEX_LOCK(&dgraph_lock);
3931                 r = run_sulogin(msg);
3932                 MUTEX_UNLOCK(&dgraph_lock);
3933                 if (r == 0)
3934                         break;
3935 
3936                 assert(r == EALREADY || r == EBUSY);
3937 
3938                 left = 3;
3939                 while (left > 0)
3940                         left = sleep(left);
3941         }
3942 
3943         MUTEX_LOCK(&single_user_thread_lock);
3944 
3945         /*
3946          * If another single user thread has started, let it finish changing
3947          * the run level.
3948          */
3949         if (single_user_thread_count > 1) {
3950                 single_user_thread_count--;
3951                 MUTEX_UNLOCK(&single_user_thread_lock);
3952                 return (NULL);
3953         }
3954 
3955         h = libscf_handle_create_bound_loop();
3956         inst = scf_instance_create(h);
3957         prop = safe_scf_property_create(h);
3958         val = safe_scf_value_create(h);
3959         buf = startd_alloc(max_scf_fmri_size);
3960 
3961 lookup:
3962         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
3963             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
3964                 switch (scf_error()) {
3965                 case SCF_ERROR_NOT_FOUND:
3966                         r = libscf_create_self(h);
3967                         if (r == 0)
3968                                 goto lookup;
3969                         assert(r == ECONNABORTED);
3970                         /* FALLTHROUGH */
3971 
3972                 case SCF_ERROR_CONNECTION_BROKEN:
3973                         libscf_handle_rebind(h);
3974                         goto lookup;
3975 
3976                 case SCF_ERROR_INVALID_ARGUMENT:
3977                 case SCF_ERROR_CONSTRAINT_VIOLATED:
3978                 case SCF_ERROR_NOT_BOUND:
3979                 case SCF_ERROR_HANDLE_MISMATCH:
3980                 default:
3981                         bad_error("scf_handle_decode_fmri", scf_error());
3982                 }
3983         }
3984 
3985         MUTEX_LOCK(&dgraph_lock);
3986 
3987         r = scf_instance_delete_prop(inst, SCF_PG_OPTIONS_OVR,
3988             SCF_PROPERTY_MILESTONE);
3989         switch (r) {
3990         case 0:
3991         case ECANCELED:
3992                 break;
3993 
3994         case ECONNABORTED:
3995                 MUTEX_UNLOCK(&dgraph_lock);
3996                 libscf_handle_rebind(h);
3997                 goto lookup;
3998 
3999         case EPERM:
4000         case EACCES:
4001         case EROFS:
4002                 log_error(LOG_WARNING, "Could not clear temporary milestone: "
4003                     "%s.\n", strerror(r));
4004                 break;
4005 
4006         default:
4007                 bad_error("scf_instance_delete_prop", r);
4008         }
4009 
4010         MUTEX_UNLOCK(&dgraph_lock);
4011 
4012         r = libscf_get_milestone(inst, prop, val, buf, max_scf_fmri_size);
4013         switch (r) {
4014         case ECANCELED:
4015         case ENOENT:
4016         case EINVAL:
4017                 (void) strcpy(buf, "all");
4018                 /* FALLTHROUGH */
4019 
4020         case 0:
4021                 uu_warn("Returning to milestone %s.\n", buf);
4022                 break;
4023 
4024         case ECONNABORTED:
4025                 libscf_handle_rebind(h);
4026                 goto lookup;
4027 
4028         default:
4029                 bad_error("libscf_get_milestone", r);
4030         }
4031 
4032         r = dgraph_set_milestone(buf, h, B_FALSE);
4033         switch (r) {
4034         case 0:
4035         case ECONNRESET:
4036         case EALREADY:
4037         case EINVAL:
4038         case ENOENT:
4039                 break;
4040 
4041         default:
4042                 bad_error("dgraph_set_milestone", r);
4043         }
4044 
4045         /*
4046          * See graph_runlevel_changed().
4047          */
4048         MUTEX_LOCK(&dgraph_lock);
4049         utmpx_set_runlevel(target_milestone_as_runlevel(), 'S', B_TRUE);
4050         MUTEX_UNLOCK(&dgraph_lock);
4051 
4052         startd_free(buf, max_scf_fmri_size);
4053         scf_value_destroy(val);
4054         scf_property_destroy(prop);
4055         scf_instance_destroy(inst);
4056         scf_handle_destroy(h);
4057 
4058         /*
4059          * We'll give ourselves 3 seconds to respond to all of the enablings
4060          * that setting the milestone should have created before checking
4061          * whether to run sulogin.
4062          */
4063         left = 3;
4064         while (left > 0)
4065                 left = sleep(left);
4066 
4067         MUTEX_LOCK(&dgraph_lock);
4068         /*
4069          * Clearing these variables will allow the sulogin thread to run.  We
4070          * check here in case there aren't any more state updates anytime soon.
4071          */
4072         go_to_level1 = go_single_user_mode = booting_to_single_user = B_FALSE;
4073         if (!sulogin_thread_running && !can_come_up()) {
4074                 (void) startd_thread_create(sulogin_thread, NULL);
4075                 sulogin_thread_running = B_TRUE;
4076         }
4077         MUTEX_UNLOCK(&dgraph_lock);
4078         single_user_thread_count--;
4079         MUTEX_UNLOCK(&single_user_thread_lock);
4080         return (NULL);
4081 }
4082 
4083 
4084 /*
4085  * Dependency graph operations API.  These are handle-independent thread-safe
4086  * graph manipulation functions which are the entry points for the event
4087  * threads below.
4088  */
4089 
4090 /*
4091  * If a configured vertex exists for inst_fmri, return EEXIST.  If no vertex
4092  * exists for inst_fmri, add one.  Then fetch the restarter from inst, make
4093  * this vertex dependent on it, and send _ADD_INSTANCE to the restarter.
4094  * Fetch whether the instance should be enabled from inst and send _ENABLE or
4095  * _DISABLE as appropriate.  Finally rummage through inst's dependency
4096  * property groups and add vertices and edges as appropriate.  If anything
4097  * goes wrong after sending _ADD_INSTANCE, send _ADMIN_MAINT_ON to put the
4098  * instance in maintenance.  Don't send _START or _STOP until we get a state
4099  * update in case we're being restarted and the service is already running.
4100  *
4101  * To support booting to a milestone, we must also make sure all dependencies
4102  * encountered are configured, if they exist in the repository.
4103  *
4104  * Returns 0 on success, ECONNABORTED on repository disconnection, EINVAL if
4105  * inst_fmri is an invalid (or not canonical) FMRI, ECANCELED if inst is
4106  * deleted, or EEXIST if a configured vertex for inst_fmri already exists.
4107  */
4108 int
4109 dgraph_add_instance(const char *inst_fmri, scf_instance_t *inst,
4110     boolean_t lock_graph)
4111 {
4112         graph_vertex_t *v;
4113         int err;
4114 
4115         if (strcmp(inst_fmri, SCF_SERVICE_STARTD) == 0)
4116                 return (0);
4117 
4118         /* Check for a vertex for inst_fmri. */
4119         if (lock_graph) {
4120                 MUTEX_LOCK(&dgraph_lock);
4121         } else {
4122                 assert(MUTEX_HELD(&dgraph_lock));
4123         }
4124 
4125         v = vertex_get_by_name(inst_fmri);
4126 
4127         if (v != NULL) {
4128                 assert(v->gv_type == GVT_INST);
4129 
4130                 if (v->gv_flags & GV_CONFIGURED) {
4131                         if (lock_graph)
4132                                 MUTEX_UNLOCK(&dgraph_lock);
4133                         return (EEXIST);
4134                 }
4135         } else {
4136                 /* Add the vertex. */
4137                 err = graph_insert_vertex_unconfigured(inst_fmri, GVT_INST, 0,
4138                     RERR_NONE, &v);
4139                 if (err != 0) {
4140                         assert(err == EINVAL);
4141                         if (lock_graph)
4142                                 MUTEX_UNLOCK(&dgraph_lock);
4143                         return (EINVAL);
4144                 }
4145         }
4146 
4147         err = configure_vertex(v, inst);
4148 
4149         if (lock_graph)
4150                 MUTEX_UNLOCK(&dgraph_lock);
4151 
4152         return (err);
4153 }
4154 
4155 /*
4156  * Locate the vertex for this property group's instance.  If it doesn't exist
4157  * or is unconfigured, call dgraph_add_instance() & return.  Otherwise fetch
4158  * the restarter for the instance, and if it has changed, send
4159  * _REMOVE_INSTANCE to the old restarter, remove the dependency, make sure the
4160  * new restarter has a vertex, add a new dependency, and send _ADD_INSTANCE to
4161  * the new restarter.  Then fetch whether the instance should be enabled, and
4162  * if it is different from what we had, or if we changed the restarter, send
4163  * the appropriate _ENABLE or _DISABLE command.
4164  *
4165  * Returns 0 on success, ENOTSUP if the pg's parent is not an instance,
4166  * ECONNABORTED on repository disconnection, ECANCELED if the instance is
4167  * deleted, or -1 if the instance's general property group is deleted or if
4168  * its enabled property is misconfigured.
4169  */
4170 static int
4171 dgraph_update_general(scf_propertygroup_t *pg)
4172 {
4173         scf_handle_t *h;
4174         scf_instance_t *inst;
4175         char *fmri;
4176         char *restarter_fmri;
4177         graph_vertex_t *v;
4178         int err;
4179         int enabled, enabled_ovr;
4180         int oldflags;
4181 
4182         /* Find the vertex for this service */
4183         h = scf_pg_handle(pg);
4184 
4185         inst = safe_scf_instance_create(h);
4186 
4187         if (scf_pg_get_parent_instance(pg, inst) != 0) {
4188                 switch (scf_error()) {
4189                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4190                         return (ENOTSUP);
4191 
4192                 case SCF_ERROR_CONNECTION_BROKEN:
4193                 default:
4194                         return (ECONNABORTED);
4195 
4196                 case SCF_ERROR_DELETED:
4197                         return (0);
4198 
4199                 case SCF_ERROR_NOT_SET:
4200                         bad_error("scf_pg_get_parent_instance", scf_error());
4201                 }
4202         }
4203 
4204         err = libscf_instance_get_fmri(inst, &fmri);
4205         switch (err) {
4206         case 0:
4207                 break;
4208 
4209         case ECONNABORTED:
4210                 scf_instance_destroy(inst);
4211                 return (ECONNABORTED);
4212 
4213         case ECANCELED:
4214                 scf_instance_destroy(inst);
4215                 return (0);
4216 
4217         default:
4218                 bad_error("libscf_instance_get_fmri", err);
4219         }
4220 
4221         log_framework(LOG_DEBUG,
4222             "Graph engine: Reloading general properties for %s.\n", fmri);
4223 
4224         MUTEX_LOCK(&dgraph_lock);
4225 
4226         v = vertex_get_by_name(fmri);
4227         if (v == NULL || !(v->gv_flags & GV_CONFIGURED)) {
4228                 /* Will get the up-to-date properties. */
4229                 MUTEX_UNLOCK(&dgraph_lock);
4230                 err = dgraph_add_instance(fmri, inst, B_TRUE);
4231                 startd_free(fmri, max_scf_fmri_size);
4232                 scf_instance_destroy(inst);
4233                 return (err == ECANCELED ? 0 : err);
4234         }
4235 
4236         /* Read enabled & restarter from repository. */
4237         restarter_fmri = startd_alloc(max_scf_value_size);
4238         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
4239             &enabled_ovr, &restarter_fmri);
4240         if (err != 0 || enabled == -1) {
4241                 MUTEX_UNLOCK(&dgraph_lock);
4242                 scf_instance_destroy(inst);
4243                 startd_free(fmri, max_scf_fmri_size);
4244 
4245                 switch (err) {
4246                 case ENOENT:
4247                 case 0:
4248                         startd_free(restarter_fmri, max_scf_value_size);
4249                         return (-1);
4250 
4251                 case ECONNABORTED:
4252                 case ECANCELED:
4253                         startd_free(restarter_fmri, max_scf_value_size);
4254                         return (err);
4255 
4256                 default:
4257                         bad_error("libscf_get_basic_instance_data", err);
4258                 }
4259         }
4260 
4261         oldflags = v->gv_flags;
4262         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
4263             (enabled ? GV_ENBLD_NOOVR : 0);
4264 
4265         if (enabled_ovr != -1)
4266                 enabled = enabled_ovr;
4267 
4268         /*
4269          * If GV_ENBLD_NOOVR has changed, then we need to re-evaluate the
4270          * subgraph.
4271          */
4272         if (milestone > MILESTONE_NONE && v->gv_flags != oldflags)
4273                 (void) eval_subgraph(v, h);
4274 
4275         scf_instance_destroy(inst);
4276 
4277         /* Ignore restarter change for now. */
4278 
4279         startd_free(restarter_fmri, max_scf_value_size);
4280         startd_free(fmri, max_scf_fmri_size);
4281 
4282         /*
4283          * Always send _ENABLE or _DISABLE.  We could avoid this if the
4284          * restarter didn't change and the enabled value didn't change, but
4285          * that's not easy to check and improbable anyway, so we'll just do
4286          * this.
4287          */
4288         graph_enable_by_vertex(v, enabled, 1);
4289 
4290         MUTEX_UNLOCK(&dgraph_lock);
4291 
4292         return (0);
4293 }
4294 
4295 /*
4296  * Delete all of the property group dependencies of v, update inst's running
4297  * snapshot, and add the dependencies in the new snapshot.  If any of the new
4298  * dependencies would create a cycle, send _ADMIN_MAINT_ON.  Otherwise
4299  * reevaluate v's dependencies, send _START or _STOP as appropriate, and do
4300  * the same for v's dependents.
4301  *
4302  * Returns
4303  *   0 - success
4304  *   ECONNABORTED - repository connection broken
4305  *   ECANCELED - inst was deleted
4306  *   EINVAL - inst is invalid (e.g., missing general/enabled)
4307  *   -1 - libscf_snapshots_refresh() failed
4308  */
4309 static int
4310 dgraph_refresh_instance(graph_vertex_t *v, scf_instance_t *inst)
4311 {
4312         int r;
4313         int enabled;
4314         int32_t tset;
4315 
4316         assert(MUTEX_HELD(&dgraph_lock));
4317         assert(v->gv_type == GVT_INST);
4318 
4319         /* Only refresh services with valid general/enabled properties. */
4320         r = libscf_get_basic_instance_data(scf_instance_handle(inst), inst,
4321             v->gv_name, &enabled, NULL, NULL);
4322         switch (r) {
4323         case 0:
4324                 break;
4325 
4326         case ECONNABORTED:
4327         case ECANCELED:
4328                 return (r);
4329 
4330         case ENOENT:
4331                 log_framework(LOG_DEBUG,
4332                     "Ignoring %s because it has no general property group.\n",
4333                     v->gv_name);
4334                 return (EINVAL);
4335 
4336         default:
4337                 bad_error("libscf_get_basic_instance_data", r);
4338         }
4339 
4340         if ((tset = libscf_get_stn_tset(inst)) == -1) {
4341                 log_framework(LOG_WARNING,
4342                     "Failed to get notification parameters for %s: %s\n",
4343                     v->gv_name, scf_strerror(scf_error()));
4344                 tset = 0;
4345         }
4346         v->gv_stn_tset = tset;
4347         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
4348                 stn_global = tset;
4349 
4350         if (enabled == -1)
4351                 return (EINVAL);
4352 
4353         r = libscf_snapshots_refresh(inst, v->gv_name);
4354         if (r != 0) {
4355                 if (r != -1)
4356                         bad_error("libscf_snapshots_refresh", r);
4357 
4358                 /* error logged */
4359                 return (r);
4360         }
4361 
4362         r = refresh_vertex(v, inst);
4363         if (r != 0 && r != ECONNABORTED)
4364                 bad_error("refresh_vertex", r);
4365         return (r);
4366 }
4367 
4368 /*
4369  * Returns true only if none of this service's dependents are 'up' -- online
4370  * or degraded (offline is considered down in this situation). This function
4371  * is somehow similar to is_nonsubgraph_leaf() but works on subtrees.
4372  */
4373 static boolean_t
4374 insubtree_dependents_down(graph_vertex_t *v)
4375 {
4376         graph_vertex_t *vv;
4377         graph_edge_t *e;
4378 
4379         assert(MUTEX_HELD(&dgraph_lock));
4380 
4381         for (e = uu_list_first(v->gv_dependents); e != NULL;
4382             e = uu_list_next(v->gv_dependents, e)) {
4383                 vv = e->ge_vertex;
4384                 if (vv->gv_type == GVT_INST) {
4385                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4386                                 continue;
4387 
4388                         if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4389                                 continue;
4390 
4391                         if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4392                             (vv->gv_state == RESTARTER_STATE_DEGRADED))
4393                                 return (B_FALSE);
4394                 } else {
4395                         /*
4396                          * Skip all excluded and optional_all dependencies
4397                          * and decide whether to offline the service based
4398                          * on restart_on attribute.
4399                          */
4400                         if (is_depgrp_bypassed(vv))
4401                                 continue;
4402 
4403                         /*
4404                          * For dependency groups or service vertices, keep
4405                          * traversing to see if instances are running.
4406                          */
4407                         if (insubtree_dependents_down(vv) == B_FALSE)
4408                                 return (B_FALSE);
4409                 }
4410         }
4411 
4412         return (B_TRUE);
4413 }
4414 
4415 /*
4416  * Returns true only if none of this service's dependents are 'up' -- online,
4417  * degraded, or offline.
4418  */
4419 static int
4420 is_nonsubgraph_leaf(graph_vertex_t *v)
4421 {
4422         graph_vertex_t *vv;
4423         graph_edge_t *e;
4424 
4425         assert(MUTEX_HELD(&dgraph_lock));
4426 
4427         for (e = uu_list_first(v->gv_dependents);
4428             e != NULL;
4429             e = uu_list_next(v->gv_dependents, e)) {
4430 
4431                 vv = e->ge_vertex;
4432                 if (vv->gv_type == GVT_INST) {
4433                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4434                                 continue;
4435 
4436                         if (vv->gv_flags & GV_INSUBGRAPH)
4437                                 continue;
4438 
4439                         if (up_state(vv->gv_state))
4440                                 return (0);
4441                 } else {
4442                         /*
4443                          * For dependency group or service vertices, keep
4444                          * traversing to see if instances are running.
4445                          *
4446                          * We should skip exclude_all dependencies otherwise
4447                          * the vertex will never be considered as a leaf
4448                          * if the dependent is offline. The main reason for
4449                          * this is that disable_nonsubgraph_leaves() skips
4450                          * exclusion dependencies.
4451                          */
4452                         if (vv->gv_type == GVT_GROUP &&
4453                             vv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4454                                 continue;
4455 
4456                         if (!is_nonsubgraph_leaf(vv))
4457                                 return (0);
4458                 }
4459         }
4460 
4461         return (1);
4462 }
4463 
4464 /*
4465  * Disable v temporarily.  Attempt to do this by setting its enabled override
4466  * property in the repository.  If that fails, send a _DISABLE command.
4467  * Returns 0 on success and ECONNABORTED if the repository connection is
4468  * broken.
4469  */
4470 static int
4471 disable_service_temporarily(graph_vertex_t *v, scf_handle_t *h)
4472 {
4473         const char * const emsg = "Could not temporarily disable %s because "
4474             "%s.  Will stop service anyways.  Repository status for the "
4475             "service may be inaccurate.\n";
4476         const char * const emsg_cbroken =
4477             "the repository connection was broken";
4478 
4479         scf_instance_t *inst;
4480         int r;
4481 
4482         inst = scf_instance_create(h);
4483         if (inst == NULL) {
4484                 char buf[100];
4485 
4486                 (void) snprintf(buf, sizeof (buf),
4487                     "scf_instance_create() failed (%s)",
4488                     scf_strerror(scf_error()));
4489                 log_error(LOG_WARNING, emsg, v->gv_name, buf);
4490 
4491                 graph_enable_by_vertex(v, 0, 0);
4492                 return (0);
4493         }
4494 
4495         r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
4496             NULL, NULL, SCF_DECODE_FMRI_EXACT);
4497         if (r != 0) {
4498                 switch (scf_error()) {
4499                 case SCF_ERROR_CONNECTION_BROKEN:
4500                         log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4501                         graph_enable_by_vertex(v, 0, 0);
4502                         return (ECONNABORTED);
4503 
4504                 case SCF_ERROR_NOT_FOUND:
4505                         return (0);
4506 
4507                 case SCF_ERROR_HANDLE_MISMATCH:
4508                 case SCF_ERROR_INVALID_ARGUMENT:
4509                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4510                 case SCF_ERROR_NOT_BOUND:
4511                 default:
4512                         bad_error("scf_handle_decode_fmri",
4513                             scf_error());
4514                 }
4515         }
4516 
4517         r = libscf_set_enable_ovr(inst, 0);
4518         switch (r) {
4519         case 0:
4520                 scf_instance_destroy(inst);
4521                 return (0);
4522 
4523         case ECANCELED:
4524                 scf_instance_destroy(inst);
4525                 return (0);
4526 
4527         case ECONNABORTED:
4528                 log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4529                 graph_enable_by_vertex(v, 0, 0);
4530                 return (ECONNABORTED);
4531 
4532         case EPERM:
4533                 log_error(LOG_WARNING, emsg, v->gv_name,
4534                     "the repository denied permission");
4535                 graph_enable_by_vertex(v, 0, 0);
4536                 return (0);
4537 
4538         case EROFS:
4539                 log_error(LOG_WARNING, emsg, v->gv_name,
4540                     "the repository is read-only");
4541                 graph_enable_by_vertex(v, 0, 0);
4542                 return (0);
4543 
4544         default:
4545                 bad_error("libscf_set_enable_ovr", r);
4546                 /* NOTREACHED */
4547         }
4548 }
4549 
4550 /*
4551  * Of the transitive instance dependencies of v, offline those which are
4552  * in the subtree and which are leaves (i.e., have no dependents which are
4553  * "up").
4554  */
4555 void
4556 offline_subtree_leaves(graph_vertex_t *v, void *arg)
4557 {
4558         assert(MUTEX_HELD(&dgraph_lock));
4559 
4560         /* If v isn't an instance, recurse on its dependencies. */
4561         if (v->gv_type != GVT_INST) {
4562                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4563                 return;
4564         }
4565 
4566         /*
4567          * If v is not in the subtree, so should all of its dependencies,
4568          * so do nothing.
4569          */
4570         if ((v->gv_flags & GV_TOOFFLINE) == 0)
4571                 return;
4572 
4573         /* If v isn't a leaf because it's already down, recurse. */
4574         if (!up_state(v->gv_state)) {
4575                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4576                 return;
4577         }
4578 
4579         /* if v is a leaf, offline it or disable it if it's the last one */
4580         if (insubtree_dependents_down(v) == B_TRUE) {
4581                 if (v->gv_flags & GV_TODISABLE)
4582                         vertex_send_event(v,
4583                             RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
4584                 else
4585                         offline_vertex(v);
4586         }
4587 }
4588 
4589 void
4590 graph_offline_subtree_leaves(graph_vertex_t *v, void *h)
4591 {
4592         graph_walk_dependencies(v, offline_subtree_leaves, (void *)h);
4593 }
4594 
4595 
4596 /*
4597  * Of the transitive instance dependencies of v, disable those which are not
4598  * in the subgraph and which are leaves (i.e., have no dependents which are
4599  * "up").
4600  */
4601 static void
4602 disable_nonsubgraph_leaves(graph_vertex_t *v, void *arg)
4603 {
4604         assert(MUTEX_HELD(&dgraph_lock));
4605 
4606         /*
4607          * We must skip exclusion dependencies because they are allowed to
4608          * complete dependency cycles.  This is correct because A's exclusion
4609          * dependency on B doesn't bear on the order in which they should be
4610          * stopped.  Indeed, the exclusion dependency should guarantee that
4611          * they are never online at the same time.
4612          */
4613         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4614                 return;
4615 
4616         /* If v isn't an instance, recurse on its dependencies. */
4617         if (v->gv_type != GVT_INST)
4618                 goto recurse;
4619 
4620         if ((v->gv_flags & GV_CONFIGURED) == 0)
4621                 /*
4622                  * Unconfigured instances should have no dependencies, but in
4623                  * case they ever get them,
4624                  */
4625                 goto recurse;
4626 
4627         /*
4628          * If v is in the subgraph, so should all of its dependencies, so do
4629          * nothing.
4630          */
4631         if (v->gv_flags & GV_INSUBGRAPH)
4632                 return;
4633 
4634         /* If v isn't a leaf because it's already down, recurse. */
4635         if (!up_state(v->gv_state))
4636                 goto recurse;
4637 
4638         /* If v is disabled but not down yet, be patient. */
4639         if ((v->gv_flags & GV_ENABLED) == 0)
4640                 return;
4641 
4642         /* If v is a leaf, disable it. */
4643         if (is_nonsubgraph_leaf(v))
4644                 (void) disable_service_temporarily(v, (scf_handle_t *)arg);
4645 
4646         return;
4647 
4648 recurse:
4649         graph_walk_dependencies(v, disable_nonsubgraph_leaves, arg);
4650 }
4651 
4652 static int
4653 stn_restarter_state(restarter_instance_state_t rstate)
4654 {
4655         static const struct statemap {
4656                 restarter_instance_state_t restarter_state;
4657                 int scf_state;
4658         } map[] = {
4659                 { RESTARTER_STATE_UNINIT, SCF_STATE_UNINIT },
4660                 { RESTARTER_STATE_MAINT, SCF_STATE_MAINT },
4661                 { RESTARTER_STATE_OFFLINE, SCF_STATE_OFFLINE },
4662                 { RESTARTER_STATE_DISABLED, SCF_STATE_DISABLED },
4663                 { RESTARTER_STATE_ONLINE, SCF_STATE_ONLINE },
4664                 { RESTARTER_STATE_DEGRADED, SCF_STATE_DEGRADED }
4665         };
4666 
4667         int i;
4668 
4669         for (i = 0; i < sizeof (map) / sizeof (map[0]); i++) {
4670                 if (rstate == map[i].restarter_state)
4671                         return (map[i].scf_state);
4672         }
4673 
4674         return (-1);
4675 }
4676 
4677 /*
4678  * State transition counters
4679  * Not incremented atomically - indicative only
4680  */
4681 static uint64_t stev_ct_maint;
4682 static uint64_t stev_ct_hwerr;
4683 static uint64_t stev_ct_service;
4684 static uint64_t stev_ct_global;
4685 static uint64_t stev_ct_noprefs;
4686 static uint64_t stev_ct_from_uninit;
4687 static uint64_t stev_ct_bad_state;
4688 static uint64_t stev_ct_ovr_prefs;
4689 
4690 static void
4691 dgraph_state_transition_notify(graph_vertex_t *v,
4692     restarter_instance_state_t old_state, restarter_str_t reason)
4693 {
4694         restarter_instance_state_t new_state = v->gv_state;
4695         int stn_transition, maint;
4696         int from, to;
4697         nvlist_t *attr;
4698         fmev_pri_t pri = FMEV_LOPRI;
4699         int raise = 0;
4700 
4701         if ((from = stn_restarter_state(old_state)) == -1 ||
4702             (to = stn_restarter_state(new_state)) == -1) {
4703                 stev_ct_bad_state++;
4704                 return;
4705         }
4706 
4707         stn_transition = from << 16 | to;
4708 
4709         maint = (to == SCF_STATE_MAINT || from == SCF_STATE_MAINT);
4710 
4711         if (maint) {
4712                 /*
4713                  * All transitions to/from maintenance state must raise
4714                  * an event.
4715                  */
4716                 raise++;
4717                 pri = FMEV_HIPRI;
4718                 stev_ct_maint++;
4719         } else if (reason == restarter_str_ct_ev_hwerr) {
4720                 /*
4721                  * All transitions caused by hardware fault must raise
4722                  * an event
4723                  */
4724                 raise++;
4725                 pri = FMEV_HIPRI;
4726                 stev_ct_hwerr++;
4727         } else if (stn_transition & v->gv_stn_tset) {
4728                 /*
4729                  * Specifically enabled event.
4730                  */
4731                 raise++;
4732                 stev_ct_service++;
4733         } else if (from == SCF_STATE_UNINIT) {
4734                 /*
4735                  * Only raise these if specifically selected above.
4736                  */
4737                 stev_ct_from_uninit++;
4738         } else if (stn_transition & stn_global &&
4739             (IS_ENABLED(v) == 1 || to == SCF_STATE_DISABLED)) {
4740                 raise++;
4741                 stev_ct_global++;
4742         } else {
4743                 stev_ct_noprefs++;
4744         }
4745 
4746         if (info_events_all) {
4747                 stev_ct_ovr_prefs++;
4748                 raise++;
4749         }
4750         if (!raise)
4751                 return;
4752 
4753         if (nvlist_alloc(&attr, NV_UNIQUE_NAME, 0) != 0 ||
4754             nvlist_add_string(attr, "fmri", v->gv_name) != 0 ||
4755             nvlist_add_uint32(attr, "reason-version",
4756             restarter_str_version()) || nvlist_add_string(attr, "reason-short",
4757             restarter_get_str_short(reason)) != 0 ||
4758             nvlist_add_string(attr, "reason-long",
4759             restarter_get_str_long(reason)) != 0 ||
4760             nvlist_add_int32(attr, "transition", stn_transition) != 0) {
4761                 log_framework(LOG_WARNING,
4762                     "FMEV: %s could not create nvlist for transition "
4763                     "event: %s\n", v->gv_name, strerror(errno));
4764                 nvlist_free(attr);
4765                 return;
4766         }
4767 
4768         if (fmev_rspublish_nvl(FMEV_RULESET_SMF, "state-transition",
4769             instance_state_str[new_state], pri, attr) != FMEV_SUCCESS) {
4770                 log_framework(LOG_DEBUG,
4771                     "FMEV: %s failed to publish transition event: %s\n",
4772                     v->gv_name, fmev_strerror(fmev_errno));
4773                 nvlist_free(attr);
4774         }
4775 }
4776 
4777 /*
4778  * Find the vertex for inst_name.  If it doesn't exist, return ENOENT.
4779  * Otherwise set its state to state.  If the instance has entered a state
4780  * which requires automatic action, take it (Uninitialized: do
4781  * dgraph_refresh_instance() without the snapshot update.  Disabled: if the
4782  * instance should be enabled, send _ENABLE.  Offline: if the instance should
4783  * be disabled, send _DISABLE, and if its dependencies are satisfied, send
4784  * _START.  Online, Degraded: if the instance wasn't running, update its start
4785  * snapshot.  Maintenance: no action.)
4786  *
4787  * Also fails with ECONNABORTED, or EINVAL if state is invalid.
4788  */
4789 static int
4790 dgraph_set_instance_state(scf_handle_t *h, const char *inst_name,
4791     protocol_states_t *states)
4792 {
4793         graph_vertex_t *v;
4794         int err = 0;
4795         restarter_instance_state_t old_state;
4796         restarter_instance_state_t state = states->ps_state;
4797         restarter_error_t serr = states->ps_err;
4798 
4799         MUTEX_LOCK(&dgraph_lock);
4800 
4801         v = vertex_get_by_name(inst_name);
4802         if (v == NULL) {
4803                 MUTEX_UNLOCK(&dgraph_lock);
4804                 return (ENOENT);
4805         }
4806 
4807         assert(v->gv_type == GVT_INST);
4808 
4809         switch (state) {
4810         case RESTARTER_STATE_UNINIT:
4811         case RESTARTER_STATE_DISABLED:
4812         case RESTARTER_STATE_OFFLINE:
4813         case RESTARTER_STATE_ONLINE:
4814         case RESTARTER_STATE_DEGRADED:
4815         case RESTARTER_STATE_MAINT:
4816                 break;
4817 
4818         default:
4819                 MUTEX_UNLOCK(&dgraph_lock);
4820                 return (EINVAL);
4821         }
4822 
4823         log_framework(LOG_DEBUG, "Graph noting %s %s -> %s.\n", v->gv_name,
4824             instance_state_str[v->gv_state], instance_state_str[state]);
4825 
4826         old_state = v->gv_state;
4827         v->gv_state = state;
4828 
4829         v->gv_reason = states->ps_reason;
4830         err = gt_transition(h, v, serr, old_state);
4831         if (err == 0 && v->gv_state != old_state) {
4832                 dgraph_state_transition_notify(v, old_state, states->ps_reason);
4833         }
4834 
4835         MUTEX_UNLOCK(&dgraph_lock);
4836         return (err);
4837 }
4838 
4839 /*
4840  * Handle state changes during milestone shutdown.  See
4841  * dgraph_set_milestone().  If the repository connection is broken,
4842  * ECONNABORTED will be returned, though a _DISABLE command will be sent for
4843  * the vertex anyway.
4844  */
4845 int
4846 vertex_subgraph_dependencies_shutdown(scf_handle_t *h, graph_vertex_t *v,
4847     restarter_instance_state_t old_state)
4848 {
4849         int was_up, now_up;
4850         int ret = 0;
4851 
4852         assert(v->gv_type == GVT_INST);
4853 
4854         /* Don't care if we're not going to a milestone. */
4855         if (milestone == NULL)
4856                 return (0);
4857 
4858         /* Don't care if we already finished coming down. */
4859         if (non_subgraph_svcs == 0)
4860                 return (0);
4861 
4862         /* Don't care if the service is in the subgraph. */
4863         if (v->gv_flags & GV_INSUBGRAPH)
4864                 return (0);
4865 
4866         /*
4867          * Update non_subgraph_svcs.  It is the number of non-subgraph
4868          * services which are in online, degraded, or offline.
4869          */
4870 
4871         was_up = up_state(old_state);
4872         now_up = up_state(v->gv_state);
4873 
4874         if (!was_up && now_up) {
4875                 ++non_subgraph_svcs;
4876         } else if (was_up && !now_up) {
4877                 --non_subgraph_svcs;
4878 
4879                 if (non_subgraph_svcs == 0) {
4880                         if (halting != -1) {
4881                                 do_uadmin();
4882                         } else if (go_single_user_mode || go_to_level1) {
4883                                 (void) startd_thread_create(single_user_thread,
4884                                     NULL);
4885                         }
4886                         return (0);
4887                 }
4888         }
4889 
4890         /* If this service is a leaf, it should be disabled. */
4891         if ((v->gv_flags & GV_ENABLED) && is_nonsubgraph_leaf(v)) {
4892                 int r;
4893 
4894                 r = disable_service_temporarily(v, h);
4895                 switch (r) {
4896                 case 0:
4897                         break;
4898 
4899                 case ECONNABORTED:
4900                         ret = ECONNABORTED;
4901                         break;
4902 
4903                 default:
4904                         bad_error("disable_service_temporarily", r);
4905                 }
4906         }
4907 
4908         /*
4909          * If the service just came down, propagate the disable to the newly
4910          * exposed leaves.
4911          */
4912         if (was_up && !now_up)
4913                 graph_walk_dependencies(v, disable_nonsubgraph_leaves,
4914                     (void *)h);
4915 
4916         return (ret);
4917 }
4918 
4919 /*
4920  * Decide whether to start up an sulogin thread after a service is
4921  * finished changing state.  Only need to do the full can_come_up()
4922  * evaluation if an instance is changing state, we're not halfway through
4923  * loading the thread, and we aren't shutting down or going to the single
4924  * user milestone.
4925  */
4926 void
4927 graph_transition_sulogin(restarter_instance_state_t state,
4928     restarter_instance_state_t old_state)
4929 {
4930         assert(MUTEX_HELD(&dgraph_lock));
4931 
4932         if (state != old_state && st->st_load_complete &&
4933             !go_single_user_mode && !go_to_level1 &&
4934             halting == -1) {
4935                 if (!sulogin_thread_running && !can_come_up()) {
4936                         (void) startd_thread_create(sulogin_thread, NULL);
4937                         sulogin_thread_running = B_TRUE;
4938                 }
4939         }
4940 }
4941 
4942 /*
4943  * Propagate a start, stop event, or a satisfiability event.
4944  *
4945  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4946  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4947  * full dependent graph to check for newly satisfied nodes.  This is
4948  * necessary for cases when non-direct dependents may be effected but direct
4949  * dependents may not (e.g. for optional_all evaluations, see the
4950  * propagate_satbility() comments).
4951  *
4952  * PROPAGATE_SAT should be used whenever a non-running service moves into
4953  * a state which can satisfy optional dependencies, like disabled or
4954  * maintenance.
4955  */
4956 void
4957 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4958     restarter_error_t rerr)
4959 {
4960         if (type == PROPAGATE_STOP) {
4961                 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4962         } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4963                 graph_walk_dependents(v, propagate_start, NULL);
4964 
4965                 if (type == PROPAGATE_SAT)
4966                         propagate_satbility(v);
4967         } else {
4968 #ifndef NDEBUG
4969                 uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4970                     __LINE__, type);
4971 #endif
4972                 abort();
4973         }
4974 }
4975 
4976 /*
4977  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4978  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4979  * all property group dependencies, and the dependency on the restarter,
4980  * disposing of vertices as appropriate.  If other vertices depend on this
4981  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4982  * returns 0.
4983  */
4984 static int
4985 dgraph_remove_instance(const char *fmri, scf_handle_t *h)
4986 {
4987         graph_vertex_t *v;
4988         graph_edge_t *e;
4989         uu_list_t *old_deps;
4990         int err;
4991 
4992         log_framework(LOG_DEBUG, "Graph engine: Removing %s.\n", fmri);
4993 
4994         MUTEX_LOCK(&dgraph_lock);
4995 
4996         v = vertex_get_by_name(fmri);
4997         if (v == NULL) {
4998                 MUTEX_UNLOCK(&dgraph_lock);
4999                 return (0);
5000         }
5001 
5002         /* Send restarter delete event. */
5003         if (v->gv_flags & GV_CONFIGURED)
5004                 graph_unset_restarter(v);
5005 
5006         if (milestone > MILESTONE_NONE) {
5007                 /*
5008                  * Make a list of v's current dependencies so we can
5009                  * reevaluate their GV_INSUBGRAPH flags after the dependencies
5010                  * are removed.
5011                  */
5012                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
5013 
5014                 err = uu_list_walk(v->gv_dependencies,
5015                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
5016                 assert(err == 0);
5017         }
5018 
5019         delete_instance_dependencies(v, B_TRUE);
5020 
5021         /*
5022          * Deleting an instance can both satisfy and unsatisfy dependencies,
5023          * depending on their type.  First propagate the stop as a RERR_RESTART
5024          * event -- deletion isn't a fault, just a normal stop.  This gives
5025          * dependent services the chance to do a clean shutdown.  Then, mark
5026          * the service as unconfigured and propagate the start event for the
5027          * optional_all dependencies that might have become satisfied.
5028          */
5029         graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
5030 
5031         v->gv_flags &= ~GV_CONFIGURED;
5032         v->gv_flags &= ~GV_DEATHROW;
5033 
5034         graph_walk_dependents(v, propagate_start, NULL);
5035         propagate_satbility(v);
5036 
5037         /*
5038          * If there are no (non-service) dependents, the vertex can be
5039          * completely removed.
5040          */
5041         if (v != milestone && v->gv_refs == 0 &&
5042             uu_list_numnodes(v->gv_dependents) == 1)
5043                 remove_inst_vertex(v);
5044 
5045         if (milestone > MILESTONE_NONE) {
5046                 void *cookie = NULL;
5047 
5048                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5049                         v = e->ge_vertex;
5050 
5051                         if (vertex_unref(v) == VERTEX_INUSE)
5052                                 while (eval_subgraph(v, h) == ECONNABORTED)
5053                                         libscf_handle_rebind(h);
5054 
5055                         startd_free(e, sizeof (*e));
5056                 }
5057 
5058                 uu_list_destroy(old_deps);
5059         }
5060 
5061         MUTEX_UNLOCK(&dgraph_lock);
5062 
5063         return (0);
5064 }
5065 
5066 /*
5067  * Return the eventual (maybe current) milestone in the form of a
5068  * legacy runlevel.
5069  */
5070 static char
5071 target_milestone_as_runlevel()
5072 {
5073         assert(MUTEX_HELD(&dgraph_lock));
5074 
5075         if (milestone == NULL)
5076                 return ('3');
5077         else if (milestone == MILESTONE_NONE)
5078                 return ('0');
5079 
5080         if (strcmp(milestone->gv_name, multi_user_fmri) == 0)
5081                 return ('2');
5082         else if (strcmp(milestone->gv_name, single_user_fmri) == 0)
5083                 return ('S');
5084         else if (strcmp(milestone->gv_name, multi_user_svr_fmri) == 0)
5085                 return ('3');
5086 
5087 #ifndef NDEBUG
5088         (void) fprintf(stderr, "%s:%d: Unknown milestone name \"%s\".\n",
5089             __FILE__, __LINE__, milestone->gv_name);
5090 #endif
5091         abort();
5092         /* NOTREACHED */
5093 }
5094 
5095 static struct {
5096         char    rl;
5097         int     sig;
5098 } init_sigs[] = {
5099         { 'S', SIGBUS },
5100         { '0', SIGINT },
5101         { '1', SIGQUIT },
5102         { '2', SIGILL },
5103         { '3', SIGTRAP },
5104         { '4', SIGIOT },
5105         { '5', SIGEMT },
5106         { '6', SIGFPE },
5107         { 0, 0 }
5108 };
5109 
5110 static void
5111 signal_init(char rl)
5112 {
5113         pid_t init_pid;
5114         int i;
5115 
5116         assert(MUTEX_HELD(&dgraph_lock));
5117 
5118         if (zone_getattr(getzoneid(), ZONE_ATTR_INITPID, &init_pid,
5119             sizeof (init_pid)) != sizeof (init_pid)) {
5120                 log_error(LOG_NOTICE, "Could not get pid to signal init.\n");
5121                 return;
5122         }
5123 
5124         for (i = 0; init_sigs[i].rl != 0; ++i)
5125                 if (init_sigs[i].rl == rl)
5126                         break;
5127 
5128         if (init_sigs[i].rl != 0) {
5129                 if (kill(init_pid, init_sigs[i].sig) != 0) {
5130                         switch (errno) {
5131                         case EPERM:
5132                         case ESRCH:
5133                                 log_error(LOG_NOTICE, "Could not signal init: "
5134                                     "%s.\n", strerror(errno));
5135                                 break;
5136 
5137                         case EINVAL:
5138                         default:
5139                                 bad_error("kill", errno);
5140                         }
5141                 }
5142         }
5143 }
5144 
5145 /*
5146  * This is called when one of the major milestones changes state, or when
5147  * init is signalled and tells us it was told to change runlevel.  We wait
5148  * to reach the milestone because this allows /etc/inittab entries to retain
5149  * some boot ordering: historically, entries could place themselves before/after
5150  * the running of /sbin/rcX scripts but we can no longer make the
5151  * distinction because the /sbin/rcX scripts no longer exist as punctuation
5152  * marks in /etc/inittab.
5153  *
5154  * Also, we only trigger an update when we reach the eventual target
5155  * milestone: without this, an /etc/inittab entry marked only for
5156  * runlevel 2 would be executed for runlevel 3, which is not how
5157  * /etc/inittab entries work.
5158  *
5159  * If we're single user coming online, then we set utmpx to the target
5160  * runlevel so that legacy scripts can work as expected.
5161  */
5162 static void
5163 graph_runlevel_changed(char rl, int online)
5164 {
5165         char trl;
5166 
5167         assert(MUTEX_HELD(&dgraph_lock));
5168 
5169         trl = target_milestone_as_runlevel();
5170 
5171         if (online) {
5172                 if (rl == trl) {
5173                         current_runlevel = trl;
5174                         signal_init(trl);
5175                 } else if (rl == 'S') {
5176                         /*
5177                          * At boot, set the entry early for the benefit of the
5178                          * legacy init scripts.
5179                          */
5180                         utmpx_set_runlevel(trl, 'S', B_FALSE);
5181                 }
5182         } else {
5183                 if (rl == '3' && trl == '2') {
5184                         current_runlevel = trl;
5185                         signal_init(trl);
5186                 } else if (rl == '2' && trl == 'S') {
5187                         current_runlevel = trl;
5188                         signal_init(trl);
5189                 }
5190         }
5191 }
5192 
5193 /*
5194  * Move to a backwards-compatible runlevel by executing the appropriate
5195  * /etc/rc?.d/K* scripts and/or setting the milestone.
5196  *
5197  * Returns
5198  *   0 - success
5199  *   ECONNRESET - success, but handle was reset
5200  *   ECONNABORTED - repository connection broken
5201  *   ECANCELED - pg was deleted
5202  */
5203 static int
5204 dgraph_set_runlevel(scf_propertygroup_t *pg, scf_property_t *prop)
5205 {
5206         char rl;
5207         scf_handle_t *h;
5208         int r;
5209         const char *ms = NULL;  /* what to commit as options/milestone */
5210         boolean_t rebound = B_FALSE;
5211         int mark_rl = 0;
5212 
5213         const char * const stop = "stop";
5214 
5215         r = libscf_extract_runlevel(prop, &rl);
5216         switch (r) {
5217         case 0:
5218                 break;
5219 
5220         case ECONNABORTED:
5221         case ECANCELED:
5222                 return (r);
5223 
5224         case EINVAL:
5225         case ENOENT:
5226                 log_error(LOG_WARNING, "runlevel property is misconfigured; "
5227                     "ignoring.\n");
5228                 /* delete the bad property */
5229                 goto nolock_out;
5230 
5231         default:
5232                 bad_error("libscf_extract_runlevel", r);
5233         }
5234 
5235         switch (rl) {
5236         case 's':
5237                 rl = 'S';
5238                 /* FALLTHROUGH */
5239 
5240         case 'S':
5241         case '2':
5242         case '3':
5243                 /*
5244                  * These cases cause a milestone change, so
5245                  * graph_runlevel_changed() will eventually deal with
5246                  * signalling init.
5247                  */
5248                 break;
5249 
5250         case '0':
5251         case '1':
5252         case '4':
5253         case '5':
5254         case '6':
5255                 mark_rl = 1;
5256                 break;
5257 
5258         default:
5259                 log_framework(LOG_NOTICE, "Unknown runlevel '%c'.\n", rl);
5260                 ms = NULL;
5261                 goto nolock_out;
5262         }
5263 
5264         h = scf_pg_handle(pg);
5265 
5266         MUTEX_LOCK(&dgraph_lock);
5267 
5268         /*
5269          * Since this triggers no milestone changes, force it by hand.
5270          */
5271         if (current_runlevel == '4' && rl == '3')
5272                 mark_rl = 1;
5273 
5274         /*
5275          * 1. If we are here after an "init X":
5276          *
5277          * init X
5278          *      init/lscf_set_runlevel()
5279          *              process_pg_event()
5280          *              dgraph_set_runlevel()
5281          *
5282          * then we haven't passed through graph_runlevel_changed() yet,
5283          * therefore 'current_runlevel' has not changed for sure but 'rl' has.
5284          * In consequence, if 'rl' is lower than 'current_runlevel', we change
5285          * the system runlevel and execute the appropriate /etc/rc?.d/K* scripts
5286          * past this test.
5287          *
5288          * 2. On the other hand, if we are here after a "svcadm milestone":
5289          *
5290          * svcadm milestone X
5291          *      dgraph_set_milestone()
5292          *              handle_graph_update_event()
5293          *              dgraph_set_instance_state()
5294          *              graph_post_X_[online|offline]()
5295          *              graph_runlevel_changed()
5296          *              signal_init()
5297          *                      init/lscf_set_runlevel()
5298          *                              process_pg_event()
5299          *                              dgraph_set_runlevel()
5300          *
5301          * then we already passed through graph_runlevel_changed() (by the way
5302          * of dgraph_set_milestone()) and 'current_runlevel' may have changed
5303          * and already be equal to 'rl' so we are going to return immediately
5304          * from dgraph_set_runlevel() without changing the system runlevel and
5305          * without executing the /etc/rc?.d/K* scripts.
5306          */
5307         if (rl == current_runlevel) {
5308                 ms = NULL;
5309                 goto out;
5310         }
5311 
5312         log_framework(LOG_DEBUG, "Changing to runlevel '%c'.\n", rl);
5313 
5314         /*
5315          * Make sure stop rc scripts see the new settings via who -r.
5316          */
5317         utmpx_set_runlevel(rl, current_runlevel, B_TRUE);
5318 
5319         /*
5320          * Some run levels don't have a direct correspondence to any
5321          * milestones, so we have to signal init directly.
5322          */
5323         if (mark_rl) {
5324                 current_runlevel = rl;
5325                 signal_init(rl);
5326         }
5327 
5328         switch (rl) {
5329         case 'S':
5330                 uu_warn("The system is coming down for administration.  "
5331                     "Please wait.\n");
5332                 fork_rc_script(rl, stop, B_FALSE);
5333                 ms = single_user_fmri;
5334                 go_single_user_mode = B_TRUE;
5335                 break;
5336 
5337         case '0':
5338                 halting_time = time(NULL);
5339                 fork_rc_script(rl, stop, B_TRUE);
5340                 halting = AD_HALT;
5341                 goto uadmin;
5342 
5343         case '5':
5344                 halting_time = time(NULL);
5345                 fork_rc_script(rl, stop, B_TRUE);
5346                 halting = AD_POWEROFF;
5347                 goto uadmin;
5348 
5349         case '6':
5350                 halting_time = time(NULL);
5351                 fork_rc_script(rl, stop, B_TRUE);
5352                 if (scf_is_fastboot_default() && getzoneid() == GLOBAL_ZONEID)
5353                         halting = AD_FASTREBOOT;
5354                 else
5355                         halting = AD_BOOT;
5356 
5357 uadmin:
5358                 uu_warn("The system is coming down.  Please wait.\n");
5359                 ms = "none";
5360 
5361                 /*
5362                  * We can't wait until all services are offline since this
5363                  * thread is responsible for taking them offline.  Instead we
5364                  * set halting to the second argument for uadmin() and call
5365                  * do_uadmin() from dgraph_set_instance_state() when
5366                  * appropriate.
5367                  */
5368                 break;
5369 
5370         case '1':
5371                 if (current_runlevel != 'S') {
5372                         uu_warn("Changing to state 1.\n");
5373                         fork_rc_script(rl, stop, B_FALSE);
5374                 } else {
5375                         uu_warn("The system is coming up for administration.  "
5376                             "Please wait.\n");
5377                 }
5378                 ms = single_user_fmri;
5379                 go_to_level1 = B_TRUE;
5380                 break;
5381 
5382         case '2':
5383                 if (current_runlevel == '3' || current_runlevel == '4')
5384                         fork_rc_script(rl, stop, B_FALSE);
5385                 ms = multi_user_fmri;
5386                 break;
5387 
5388         case '3':
5389         case '4':
5390                 ms = "all";
5391                 break;
5392 
5393         default:
5394 #ifndef NDEBUG
5395                 (void) fprintf(stderr, "%s:%d: Uncaught case %d ('%c').\n",
5396                     __FILE__, __LINE__, rl, rl);
5397 #endif
5398                 abort();
5399         }
5400 
5401 out:
5402         MUTEX_UNLOCK(&dgraph_lock);
5403 
5404 nolock_out:
5405         switch (r = libscf_clear_runlevel(pg, ms)) {
5406         case 0:
5407                 break;
5408 
5409         case ECONNABORTED:
5410                 libscf_handle_rebind(h);
5411                 rebound = B_TRUE;
5412                 goto nolock_out;
5413 
5414         case ECANCELED:
5415                 break;
5416 
5417         case EPERM:
5418         case EACCES:
5419         case EROFS:
5420                 log_error(LOG_NOTICE, "Could not delete \"%s/%s\" property: "
5421                     "%s.\n", SCF_PG_OPTIONS, "runlevel", strerror(r));
5422                 break;
5423 
5424         default:
5425                 bad_error("libscf_clear_runlevel", r);
5426         }
5427 
5428         return (rebound ? ECONNRESET : 0);
5429 }
5430 
5431 /*
5432  * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5433  * to the instances that are supposed to go offline during an
5434  * administrative disable operation.
5435  */
5436 static int
5437 mark_subtree(graph_edge_t *e, void *arg)
5438 {
5439         graph_vertex_t *v;
5440         int r;
5441 
5442         v = e->ge_vertex;
5443 
5444         /* If it's already in the subgraph, skip. */
5445         if (v->gv_flags & GV_TOOFFLINE)
5446                 return (UU_WALK_NEXT);
5447 
5448         switch (v->gv_type) {
5449         case GVT_INST:
5450                 /* If the instance is already disabled, skip it. */
5451                 if (!(v->gv_flags & GV_ENABLED))
5452                         return (UU_WALK_NEXT);
5453 
5454                 v->gv_flags |= GV_TOOFFLINE;
5455                 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5456                 break;
5457         case GVT_GROUP:
5458                 /*
5459                  * Skip all excluded and optional_all dependencies and decide
5460                  * whether to offline the service based on restart_on attribute.
5461                  */
5462                 if (is_depgrp_bypassed(v))
5463                         return (UU_WALK_NEXT);
5464                 break;
5465         }
5466 
5467         r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5468             0);
5469         assert(r == 0);
5470         return (UU_WALK_NEXT);
5471 }
5472 
5473 static int
5474 mark_subgraph(graph_edge_t *e, void *arg)
5475 {
5476         graph_vertex_t *v;
5477         int r;
5478         int optional = (int)arg;
5479 
5480         v = e->ge_vertex;
5481 
5482         /* If it's already in the subgraph, skip. */
5483         if (v->gv_flags & GV_INSUBGRAPH)
5484                 return (UU_WALK_NEXT);
5485 
5486         /*
5487          * Keep track if walk has entered an optional dependency group
5488          */
5489         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_OPTIONAL_ALL) {
5490                 optional = 1;
5491         }
5492         /*
5493          * Quit if we are in an optional dependency group and the instance
5494          * is disabled
5495          */
5496         if (optional && (v->gv_type == GVT_INST) &&
5497             (!(v->gv_flags & GV_ENBLD_NOOVR)))
5498                 return (UU_WALK_NEXT);
5499 
5500         v->gv_flags |= GV_INSUBGRAPH;
5501 
5502         /* Skip all excluded dependencies. */
5503         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
5504                 return (UU_WALK_NEXT);
5505 
5506         r = uu_list_walk(v->gv_dependencies, (uu_walk_fn_t *)mark_subgraph,
5507             (void *)optional, 0);
5508         assert(r == 0);
5509         return (UU_WALK_NEXT);
5510 }
5511 
5512 /*
5513  * Bring down all services which are not dependencies of fmri.  The
5514  * dependencies of fmri (direct & indirect) will constitute the "subgraph",
5515  * and will have the GV_INSUBGRAPH flag set.  The rest must be brought down,
5516  * which means the state is "disabled", "maintenance", or "uninitialized".  We
5517  * could consider "offline" to be down, and refrain from sending start
5518  * commands for such services, but that's not strictly necessary, so we'll
5519  * decline to intrude on the state machine.  It would probably confuse users
5520  * anyway.
5521  *
5522  * The services should be brought down in reverse-dependency order, so we
5523  * can't do it all at once here.  We initiate by override-disabling the leaves
5524  * of the dependency tree -- those services which are up but have no
5525  * dependents which are up.  When they come down,
5526  * vertex_subgraph_dependencies_shutdown() will override-disable the newly
5527  * exposed leaves.  Perseverance will ensure completion.
5528  *
5529  * Sometimes we need to take action when the transition is complete, like
5530  * start sulogin or halt the system.  To tell when we're done, we initialize
5531  * non_subgraph_svcs here to be the number of services which need to come
5532  * down.  As each does, we decrement the counter.  When it hits zero, we take
5533  * the appropriate action.  See vertex_subgraph_dependencies_shutdown().
5534  *
5535  * In case we're coming up, we also remove any enable-overrides for the
5536  * services which are dependencies of fmri.
5537  *
5538  * If norepository is true, the function will not change the repository.
5539  *
5540  * The decision to change the system run level in accordance with the milestone
5541  * is taken in dgraph_set_runlevel().
5542  *
5543  * Returns
5544  *   0 - success
5545  *   ECONNRESET - success, but handle was rebound
5546  *   EINVAL - fmri is invalid (error is logged)
5547  *   EALREADY - the milestone is already set to fmri
5548  *   ENOENT - a configured vertex does not exist for fmri (an error is logged)
5549  */
5550 static int
5551 dgraph_set_milestone(const char *fmri, scf_handle_t *h, boolean_t norepository)
5552 {
5553         const char *cfmri, *fs;
5554         graph_vertex_t *nm, *v;
5555         int ret = 0, r;
5556         scf_instance_t *inst;
5557         boolean_t isall, isnone, rebound = B_FALSE;
5558 
5559         /* Validate fmri */
5560         isall = (strcmp(fmri, "all") == 0);
5561         isnone = (strcmp(fmri, "none") == 0);
5562 
5563         if (!isall && !isnone) {
5564                 if (fmri_canonify(fmri, (char **)&cfmri, B_FALSE) == EINVAL)
5565                         goto reject;
5566 
5567                 if (strcmp(cfmri, single_user_fmri) != 0 &&
5568                     strcmp(cfmri, multi_user_fmri) != 0 &&
5569                     strcmp(cfmri, multi_user_svr_fmri) != 0) {
5570                         startd_free((void *)cfmri, max_scf_fmri_size);
5571 reject:
5572                         log_framework(LOG_WARNING,
5573                             "Rejecting request for invalid milestone \"%s\".\n",
5574                             fmri);
5575                         return (EINVAL);
5576                 }
5577         }
5578 
5579         inst = safe_scf_instance_create(h);
5580 
5581         MUTEX_LOCK(&dgraph_lock);
5582 
5583         if (milestone == NULL) {
5584                 if (isall) {
5585                         log_framework(LOG_DEBUG,
5586                             "Milestone already set to all.\n");
5587                         ret = EALREADY;
5588                         goto out;
5589                 }
5590         } else if (milestone == MILESTONE_NONE) {
5591                 if (isnone) {
5592                         log_framework(LOG_DEBUG,
5593                             "Milestone already set to none.\n");
5594                         ret = EALREADY;
5595                         goto out;
5596                 }
5597         } else {
5598                 if (!isall && !isnone &&
5599                     strcmp(cfmri, milestone->gv_name) == 0) {
5600                         log_framework(LOG_DEBUG,
5601                             "Milestone already set to %s.\n", cfmri);
5602                         ret = EALREADY;
5603                         goto out;
5604                 }
5605         }
5606 
5607         if (!isall && !isnone) {
5608                 nm = vertex_get_by_name(cfmri);
5609                 if (nm == NULL || !(nm->gv_flags & GV_CONFIGURED)) {
5610                         log_framework(LOG_WARNING, "Cannot set milestone to %s "
5611                             "because no such service exists.\n", cfmri);
5612                         ret = ENOENT;
5613                         goto out;
5614                 }
5615         }
5616 
5617         log_framework(LOG_DEBUG, "Changing milestone to %s.\n", fmri);
5618 
5619         /*
5620          * Set milestone, removing the old one if this was the last reference.
5621          */
5622         if (milestone > MILESTONE_NONE)
5623                 (void) vertex_unref(milestone);
5624 
5625         if (isall)
5626                 milestone = NULL;
5627         else if (isnone)
5628                 milestone = MILESTONE_NONE;
5629         else {
5630                 milestone = nm;
5631                 /* milestone should count as a reference */
5632                 vertex_ref(milestone);
5633         }
5634 
5635         /* Clear all GV_INSUBGRAPH bits. */
5636         for (v = uu_list_first(dgraph); v != NULL; v = uu_list_next(dgraph, v))
5637                 v->gv_flags &= ~GV_INSUBGRAPH;
5638 
5639         if (!isall && !isnone) {
5640                 /* Set GV_INSUBGRAPH for milestone & descendents. */
5641                 milestone->gv_flags |= GV_INSUBGRAPH;
5642 
5643                 r = uu_list_walk(milestone->gv_dependencies,
5644                     (uu_walk_fn_t *)mark_subgraph, NULL, 0);
5645                 assert(r == 0);
5646         }
5647 
5648         /* Un-override services in the subgraph & override-disable the rest. */
5649         if (norepository)
5650                 goto out;
5651 
5652         non_subgraph_svcs = 0;
5653         for (v = uu_list_first(dgraph);
5654             v != NULL;
5655             v = uu_list_next(dgraph, v)) {
5656                 if (v->gv_type != GVT_INST ||
5657                     (v->gv_flags & GV_CONFIGURED) == 0)
5658                         continue;
5659 
5660 again:
5661                 r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
5662                     NULL, NULL, SCF_DECODE_FMRI_EXACT);
5663                 if (r != 0) {
5664                         switch (scf_error()) {
5665                         case SCF_ERROR_CONNECTION_BROKEN:
5666                         default:
5667                                 libscf_handle_rebind(h);
5668                                 rebound = B_TRUE;
5669                                 goto again;
5670 
5671                         case SCF_ERROR_NOT_FOUND:
5672                                 continue;
5673 
5674                         case SCF_ERROR_HANDLE_MISMATCH:
5675                         case SCF_ERROR_INVALID_ARGUMENT:
5676                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5677                         case SCF_ERROR_NOT_BOUND:
5678                                 bad_error("scf_handle_decode_fmri",
5679                                     scf_error());
5680                         }
5681                 }
5682 
5683                 if (isall || (v->gv_flags & GV_INSUBGRAPH)) {
5684                         r = libscf_delete_enable_ovr(inst);
5685                         fs = "libscf_delete_enable_ovr";
5686                 } else {
5687                         assert(isnone || (v->gv_flags & GV_INSUBGRAPH) == 0);
5688 
5689                         /*
5690                          * Services which are up need to come down before
5691                          * we're done, but we can only disable the leaves
5692                          * here.
5693                          */
5694 
5695                         if (up_state(v->gv_state))
5696                                 ++non_subgraph_svcs;
5697 
5698                         /* If it's already disabled, don't bother. */
5699                         if ((v->gv_flags & GV_ENABLED) == 0)
5700                                 continue;
5701 
5702                         if (!is_nonsubgraph_leaf(v))
5703                                 continue;
5704 
5705                         r = libscf_set_enable_ovr(inst, 0);
5706                         fs = "libscf_set_enable_ovr";
5707                 }
5708                 switch (r) {
5709                 case 0:
5710                 case ECANCELED:
5711                         break;
5712 
5713                 case ECONNABORTED:
5714                         libscf_handle_rebind(h);
5715                         rebound = B_TRUE;
5716                         goto again;
5717 
5718                 case EPERM:
5719                 case EROFS:
5720                         log_error(LOG_WARNING,
5721                             "Could not set %s/%s for %s: %s.\n",
5722                             SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
5723                             v->gv_name, strerror(r));
5724                         break;
5725 
5726                 default:
5727                         bad_error(fs, r);
5728                 }
5729         }
5730 
5731         if (halting != -1) {
5732                 if (non_subgraph_svcs > 1)
5733                         uu_warn("%d system services are now being stopped.\n",
5734                             non_subgraph_svcs);
5735                 else if (non_subgraph_svcs == 1)
5736                         uu_warn("One system service is now being stopped.\n");
5737                 else if (non_subgraph_svcs == 0)
5738                         do_uadmin();
5739         }
5740 
5741         ret = rebound ? ECONNRESET : 0;
5742 
5743 out:
5744         MUTEX_UNLOCK(&dgraph_lock);
5745         if (!isall && !isnone)
5746                 startd_free((void *)cfmri, max_scf_fmri_size);
5747         scf_instance_destroy(inst);
5748         return (ret);
5749 }
5750 
5751 
5752 /*
5753  * Returns 0, ECONNABORTED, or EINVAL.
5754  */
5755 static int
5756 handle_graph_update_event(scf_handle_t *h, graph_protocol_event_t *e)
5757 {
5758         int r;
5759 
5760         switch (e->gpe_type) {
5761         case GRAPH_UPDATE_RELOAD_GRAPH:
5762                 log_error(LOG_WARNING,
5763                     "graph_event: reload graph unimplemented\n");
5764                 break;
5765 
5766         case GRAPH_UPDATE_STATE_CHANGE: {
5767                 protocol_states_t *states = e->gpe_data;
5768 
5769                 switch (r = dgraph_set_instance_state(h, e->gpe_inst, states)) {
5770                 case 0:
5771                 case ENOENT:
5772                         break;
5773 
5774                 case ECONNABORTED:
5775                         return (ECONNABORTED);
5776 
5777                 case EINVAL:
5778                 default:
5779 #ifndef NDEBUG
5780                         (void) fprintf(stderr, "dgraph_set_instance_state() "
5781                             "failed with unexpected error %d at %s:%d.\n", r,
5782                             __FILE__, __LINE__);
5783 #endif
5784                         abort();
5785                 }
5786 
5787                 startd_free(states, sizeof (protocol_states_t));
5788                 break;
5789         }
5790 
5791         default:
5792                 log_error(LOG_WARNING,
5793                     "graph_event_loop received an unknown event: %d\n",
5794                     e->gpe_type);
5795                 break;
5796         }
5797 
5798         return (0);
5799 }
5800 
5801 /*
5802  * graph_event_thread()
5803  *    Wait for state changes from the restarters.
5804  */
5805 /*ARGSUSED*/
5806 void *
5807 graph_event_thread(void *unused)
5808 {
5809         scf_handle_t *h;
5810         int err;
5811 
5812         h = libscf_handle_create_bound_loop();
5813 
5814         /*CONSTCOND*/
5815         while (1) {
5816                 graph_protocol_event_t *e;
5817 
5818                 MUTEX_LOCK(&gu->gu_lock);
5819 
5820                 while (gu->gu_wakeup == 0)
5821                         (void) pthread_cond_wait(&gu->gu_cv, &gu->gu_lock);
5822 
5823                 gu->gu_wakeup = 0;
5824 
5825                 while ((e = graph_event_dequeue()) != NULL) {
5826                         MUTEX_LOCK(&e->gpe_lock);
5827                         MUTEX_UNLOCK(&gu->gu_lock);
5828 
5829                         while ((err = handle_graph_update_event(h, e)) ==
5830                             ECONNABORTED)
5831                                 libscf_handle_rebind(h);
5832 
5833                         if (err == 0)
5834                                 graph_event_release(e);
5835                         else
5836                                 graph_event_requeue(e);
5837 
5838                         MUTEX_LOCK(&gu->gu_lock);
5839                 }
5840 
5841                 MUTEX_UNLOCK(&gu->gu_lock);
5842         }
5843 
5844         /*
5845          * Unreachable for now -- there's currently no graceful cleanup
5846          * called on exit().
5847          */
5848         MUTEX_UNLOCK(&gu->gu_lock);
5849         scf_handle_destroy(h);
5850         return (NULL);
5851 }
5852 
5853 static void
5854 set_initial_milestone(scf_handle_t *h)
5855 {
5856         scf_instance_t *inst;
5857         char *fmri, *cfmri;
5858         size_t sz;
5859         int r;
5860 
5861         inst = safe_scf_instance_create(h);
5862         fmri = startd_alloc(max_scf_fmri_size);
5863 
5864         /*
5865          * If -m milestone= was specified, we want to set options_ovr/milestone
5866          * to it.  Otherwise we want to read what the milestone should be set
5867          * to.  Either way we need our inst.
5868          */
5869 get_self:
5870         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
5871             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5872                 switch (scf_error()) {
5873                 case SCF_ERROR_CONNECTION_BROKEN:
5874                         libscf_handle_rebind(h);
5875                         goto get_self;
5876 
5877                 case SCF_ERROR_NOT_FOUND:
5878                         if (st->st_subgraph != NULL &&
5879                             st->st_subgraph[0] != '\0') {
5880                                 sz = strlcpy(fmri, st->st_subgraph,
5881                                     max_scf_fmri_size);
5882                                 assert(sz < max_scf_fmri_size);
5883                         } else {
5884                                 fmri[0] = '\0';
5885                         }
5886                         break;
5887 
5888                 case SCF_ERROR_INVALID_ARGUMENT:
5889                 case SCF_ERROR_CONSTRAINT_VIOLATED:
5890                 case SCF_ERROR_HANDLE_MISMATCH:
5891                 default:
5892                         bad_error("scf_handle_decode_fmri", scf_error());
5893                 }
5894         } else {
5895                 if (st->st_subgraph != NULL && st->st_subgraph[0] != '\0') {
5896                         scf_propertygroup_t *pg;
5897 
5898                         pg = safe_scf_pg_create(h);
5899 
5900                         sz = strlcpy(fmri, st->st_subgraph, max_scf_fmri_size);
5901                         assert(sz < max_scf_fmri_size);
5902 
5903                         r = libscf_inst_get_or_add_pg(inst, SCF_PG_OPTIONS_OVR,
5904                             SCF_PG_OPTIONS_OVR_TYPE, SCF_PG_OPTIONS_OVR_FLAGS,
5905                             pg);
5906                         switch (r) {
5907                         case 0:
5908                                 break;
5909 
5910                         case ECONNABORTED:
5911                                 libscf_handle_rebind(h);
5912                                 goto get_self;
5913 
5914                         case EPERM:
5915                         case EACCES:
5916                         case EROFS:
5917                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5918                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5919                                     SCF_PROPERTY_MILESTONE, strerror(r));
5920                                 /* FALLTHROUGH */
5921 
5922                         case ECANCELED:
5923                                 sz = strlcpy(fmri, st->st_subgraph,
5924                                     max_scf_fmri_size);
5925                                 assert(sz < max_scf_fmri_size);
5926                                 break;
5927 
5928                         default:
5929                                 bad_error("libscf_inst_get_or_add_pg", r);
5930                         }
5931 
5932                         r = libscf_clear_runlevel(pg, fmri);
5933                         switch (r) {
5934                         case 0:
5935                                 break;
5936 
5937                         case ECONNABORTED:
5938                                 libscf_handle_rebind(h);
5939                                 goto get_self;
5940 
5941                         case EPERM:
5942                         case EACCES:
5943                         case EROFS:
5944                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5945                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5946                                     SCF_PROPERTY_MILESTONE, strerror(r));
5947                                 /* FALLTHROUGH */
5948 
5949                         case ECANCELED:
5950                                 sz = strlcpy(fmri, st->st_subgraph,
5951                                     max_scf_fmri_size);
5952                                 assert(sz < max_scf_fmri_size);
5953                                 break;
5954 
5955                         default:
5956                                 bad_error("libscf_clear_runlevel", r);
5957                         }
5958 
5959                         scf_pg_destroy(pg);
5960                 } else {
5961                         scf_property_t *prop;
5962                         scf_value_t *val;
5963 
5964                         prop = safe_scf_property_create(h);
5965                         val = safe_scf_value_create(h);
5966 
5967                         r = libscf_get_milestone(inst, prop, val, fmri,
5968                             max_scf_fmri_size);
5969                         switch (r) {
5970                         case 0:
5971                                 break;
5972 
5973                         case ECONNABORTED:
5974                                 libscf_handle_rebind(h);
5975                                 goto get_self;
5976 
5977                         case EINVAL:
5978                                 log_error(LOG_WARNING, "Milestone property is "
5979                                     "misconfigured.  Defaulting to \"all\".\n");
5980                                 /* FALLTHROUGH */
5981 
5982                         case ECANCELED:
5983                         case ENOENT:
5984                                 fmri[0] = '\0';
5985                                 break;
5986 
5987                         default:
5988                                 bad_error("libscf_get_milestone", r);
5989                         }
5990 
5991                         scf_value_destroy(val);
5992                         scf_property_destroy(prop);
5993                 }
5994         }
5995 
5996         if (fmri[0] == '\0' || strcmp(fmri, "all") == 0)
5997                 goto out;
5998 
5999         if (strcmp(fmri, "none") != 0) {
6000 retry:
6001                 if (scf_handle_decode_fmri(h, fmri, NULL, NULL, inst, NULL,
6002                     NULL, SCF_DECODE_FMRI_EXACT) != 0) {
6003                         switch (scf_error()) {
6004                         case SCF_ERROR_INVALID_ARGUMENT:
6005                                 log_error(LOG_WARNING,
6006                                     "Requested milestone \"%s\" is invalid.  "
6007                                     "Reverting to \"all\".\n", fmri);
6008                                 goto out;
6009 
6010                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6011                                 log_error(LOG_WARNING, "Requested milestone "
6012                                     "\"%s\" does not specify an instance.  "
6013                                     "Reverting to \"all\".\n", fmri);
6014                                 goto out;
6015 
6016                         case SCF_ERROR_CONNECTION_BROKEN:
6017                                 libscf_handle_rebind(h);
6018                                 goto retry;
6019 
6020                         case SCF_ERROR_NOT_FOUND:
6021                                 log_error(LOG_WARNING, "Requested milestone "
6022                                     "\"%s\" not in repository.  Reverting to "
6023                                     "\"all\".\n", fmri);
6024                                 goto out;
6025 
6026                         case SCF_ERROR_HANDLE_MISMATCH:
6027                         default:
6028                                 bad_error("scf_handle_decode_fmri",
6029                                     scf_error());
6030                         }
6031                 }
6032 
6033                 r = fmri_canonify(fmri, &cfmri, B_FALSE);
6034                 assert(r == 0);
6035 
6036                 r = dgraph_add_instance(cfmri, inst, B_TRUE);
6037                 startd_free(cfmri, max_scf_fmri_size);
6038                 switch (r) {
6039                 case 0:
6040                         break;
6041 
6042                 case ECONNABORTED:
6043                         goto retry;
6044 
6045                 case EINVAL:
6046                         log_error(LOG_WARNING,
6047                             "Requested milestone \"%s\" is invalid.  "
6048                             "Reverting to \"all\".\n", fmri);
6049                         goto out;
6050 
6051                 case ECANCELED:
6052                         log_error(LOG_WARNING,
6053                             "Requested milestone \"%s\" not "
6054                             "in repository.  Reverting to \"all\".\n",
6055                             fmri);
6056                         goto out;
6057 
6058                 case EEXIST:
6059                 default:
6060                         bad_error("dgraph_add_instance", r);
6061                 }
6062         }
6063 
6064         log_console(LOG_INFO, "Booting to milestone \"%s\".\n", fmri);
6065 
6066         r = dgraph_set_milestone(fmri, h, B_FALSE);
6067         switch (r) {
6068         case 0:
6069         case ECONNRESET:
6070         case EALREADY:
6071                 break;
6072 
6073         case EINVAL:
6074         case ENOENT:
6075         default:
6076                 bad_error("dgraph_set_milestone", r);
6077         }
6078 
6079 out:
6080         startd_free(fmri, max_scf_fmri_size);
6081         scf_instance_destroy(inst);
6082 }
6083 
6084 void
6085 set_restart_milestone(scf_handle_t *h)
6086 {
6087         scf_instance_t *inst;
6088         scf_property_t *prop;
6089         scf_value_t *val;
6090         char *fmri;
6091         int r;
6092 
6093         inst = safe_scf_instance_create(h);
6094 
6095 get_self:
6096         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL,
6097             inst, NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
6098                 switch (scf_error()) {
6099                 case SCF_ERROR_CONNECTION_BROKEN:
6100                         libscf_handle_rebind(h);
6101                         goto get_self;
6102 
6103                 case SCF_ERROR_NOT_FOUND:
6104                         break;
6105 
6106                 case SCF_ERROR_INVALID_ARGUMENT:
6107                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6108                 case SCF_ERROR_HANDLE_MISMATCH:
6109                 default:
6110                         bad_error("scf_handle_decode_fmri", scf_error());
6111                 }
6112 
6113                 scf_instance_destroy(inst);
6114                 return;
6115         }
6116 
6117         prop = safe_scf_property_create(h);
6118         val = safe_scf_value_create(h);
6119         fmri = startd_alloc(max_scf_fmri_size);
6120 
6121         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6122         switch (r) {
6123         case 0:
6124                 break;
6125 
6126         case ECONNABORTED:
6127                 libscf_handle_rebind(h);
6128                 goto get_self;
6129 
6130         case ECANCELED:
6131         case ENOENT:
6132         case EINVAL:
6133                 goto out;
6134 
6135         default:
6136                 bad_error("libscf_get_milestone", r);
6137         }
6138 
6139         r = dgraph_set_milestone(fmri, h, B_TRUE);
6140         switch (r) {
6141         case 0:
6142         case ECONNRESET:
6143         case EALREADY:
6144         case EINVAL:
6145         case ENOENT:
6146                 break;
6147 
6148         default:
6149                 bad_error("dgraph_set_milestone", r);
6150         }
6151 
6152 out:
6153         startd_free(fmri, max_scf_fmri_size);
6154         scf_value_destroy(val);
6155         scf_property_destroy(prop);
6156         scf_instance_destroy(inst);
6157 }
6158 
6159 /*
6160  * void *graph_thread(void *)
6161  *
6162  * Graph management thread.
6163  */
6164 /*ARGSUSED*/
6165 void *
6166 graph_thread(void *arg)
6167 {
6168         scf_handle_t *h;
6169         int err;
6170 
6171         h = libscf_handle_create_bound_loop();
6172 
6173         if (st->st_initial)
6174                 set_initial_milestone(h);
6175 
6176         MUTEX_LOCK(&dgraph_lock);
6177         initial_milestone_set = B_TRUE;
6178         err = pthread_cond_broadcast(&initial_milestone_cv);
6179         assert(err == 0);
6180         MUTEX_UNLOCK(&dgraph_lock);
6181 
6182         libscf_populate_graph(h);
6183 
6184         if (!st->st_initial)
6185                 set_restart_milestone(h);
6186 
6187         MUTEX_LOCK(&st->st_load_lock);
6188         st->st_load_complete = 1;
6189         (void) pthread_cond_broadcast(&st->st_load_cv);
6190         MUTEX_UNLOCK(&st->st_load_lock);
6191 
6192         MUTEX_LOCK(&dgraph_lock);
6193         /*
6194          * Now that we've set st_load_complete we need to check can_come_up()
6195          * since if we booted to a milestone, then there won't be any more
6196          * state updates.
6197          */
6198         if (!go_single_user_mode && !go_to_level1 &&
6199             halting == -1) {
6200                 if (!sulogin_thread_running && !can_come_up()) {
6201                         (void) startd_thread_create(sulogin_thread, NULL);
6202                         sulogin_thread_running = B_TRUE;
6203                 }
6204         }
6205         MUTEX_UNLOCK(&dgraph_lock);
6206 
6207         (void) pthread_mutex_lock(&gu->gu_freeze_lock);
6208 
6209         /*CONSTCOND*/
6210         while (1) {
6211                 (void) pthread_cond_wait(&gu->gu_freeze_cv,
6212                     &gu->gu_freeze_lock);
6213         }
6214 
6215         /*
6216          * Unreachable for now -- there's currently no graceful cleanup
6217          * called on exit().
6218          */
6219         (void) pthread_mutex_unlock(&gu->gu_freeze_lock);
6220         scf_handle_destroy(h);
6221 
6222         return (NULL);
6223 }
6224 
6225 
6226 /*
6227  * int next_action()
6228  *   Given an array of timestamps 'a' with 'num' elements, find the
6229  *   lowest non-zero timestamp and return its index. If there are no
6230  *   non-zero elements, return -1.
6231  */
6232 static int
6233 next_action(hrtime_t *a, int num)
6234 {
6235         hrtime_t t = 0;
6236         int i = 0, smallest = -1;
6237 
6238         for (i = 0; i < num; i++) {
6239                 if (t == 0) {
6240                         t = a[i];
6241                         smallest = i;
6242                 } else if (a[i] != 0 && a[i] < t) {
6243                         t = a[i];
6244                         smallest = i;
6245                 }
6246         }
6247 
6248         if (t == 0)
6249                 return (-1);
6250         else
6251                 return (smallest);
6252 }
6253 
6254 /*
6255  * void process_actions()
6256  *   Process actions requested by the administrator. Possibilities include:
6257  *   refresh, restart, maintenance mode off, maintenance mode on,
6258  *   maintenance mode immediate, and degraded.
6259  *
6260  *   The set of pending actions is represented in the repository as a
6261  *   per-instance property group, with each action being a single property
6262  *   in that group.  This property group is converted to an array, with each
6263  *   action type having an array slot.  The actions in the array at the
6264  *   time process_actions() is called are acted on in the order of the
6265  *   timestamp (which is the value stored in the slot).  A value of zero
6266  *   indicates that there is no pending action of the type associated with
6267  *   a particular slot.
6268  *
6269  *   Sending an action event multiple times before the restarter has a
6270  *   chance to process that action will force it to be run at the last
6271  *   timestamp where it appears in the ordering.
6272  *
6273  *   Turning maintenance mode on trumps all other actions.
6274  *
6275  *   Returns 0 or ECONNABORTED.
6276  */
6277 static int
6278 process_actions(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst)
6279 {
6280         scf_property_t *prop = NULL;
6281         scf_value_t *val = NULL;
6282         scf_type_t type;
6283         graph_vertex_t *vertex;
6284         admin_action_t a;
6285         int i, ret = 0, r;
6286         hrtime_t action_ts[NACTIONS];
6287         char *inst_name;
6288 
6289         r = libscf_instance_get_fmri(inst, &inst_name);
6290         switch (r) {
6291         case 0:
6292                 break;
6293 
6294         case ECONNABORTED:
6295                 return (ECONNABORTED);
6296 
6297         case ECANCELED:
6298                 return (0);
6299 
6300         default:
6301                 bad_error("libscf_instance_get_fmri", r);
6302         }
6303 
6304         MUTEX_LOCK(&dgraph_lock);
6305 
6306         vertex = vertex_get_by_name(inst_name);
6307         if (vertex == NULL) {
6308                 MUTEX_UNLOCK(&dgraph_lock);
6309                 log_framework(LOG_DEBUG, "%s: Can't find graph vertex. "
6310                     "The instance must have been removed.\n", inst_name);
6311                 startd_free(inst_name, max_scf_fmri_size);
6312                 return (0);
6313         }
6314 
6315         prop = safe_scf_property_create(h);
6316         val = safe_scf_value_create(h);
6317 
6318         for (i = 0; i < NACTIONS; i++) {
6319                 if (scf_pg_get_property(pg, admin_actions[i], prop) != 0) {
6320                         switch (scf_error()) {
6321                         case SCF_ERROR_CONNECTION_BROKEN:
6322                         default:
6323                                 ret = ECONNABORTED;
6324                                 goto out;
6325 
6326                         case SCF_ERROR_DELETED:
6327                                 goto out;
6328 
6329                         case SCF_ERROR_NOT_FOUND:
6330                                 action_ts[i] = 0;
6331                                 continue;
6332 
6333                         case SCF_ERROR_HANDLE_MISMATCH:
6334                         case SCF_ERROR_INVALID_ARGUMENT:
6335                         case SCF_ERROR_NOT_SET:
6336                                 bad_error("scf_pg_get_property", scf_error());
6337                         }
6338                 }
6339 
6340                 if (scf_property_type(prop, &type) != 0) {
6341                         switch (scf_error()) {
6342                         case SCF_ERROR_CONNECTION_BROKEN:
6343                         default:
6344                                 ret = ECONNABORTED;
6345                                 goto out;
6346 
6347                         case SCF_ERROR_DELETED:
6348                                 action_ts[i] = 0;
6349                                 continue;
6350 
6351                         case SCF_ERROR_NOT_SET:
6352                                 bad_error("scf_property_type", scf_error());
6353                         }
6354                 }
6355 
6356                 if (type != SCF_TYPE_INTEGER) {
6357                         action_ts[i] = 0;
6358                         continue;
6359                 }
6360 
6361                 if (scf_property_get_value(prop, val) != 0) {
6362                         switch (scf_error()) {
6363                         case SCF_ERROR_CONNECTION_BROKEN:
6364                         default:
6365                                 ret = ECONNABORTED;
6366                                 goto out;
6367 
6368                         case SCF_ERROR_DELETED:
6369                                 goto out;
6370 
6371                         case SCF_ERROR_NOT_FOUND:
6372                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6373                                 action_ts[i] = 0;
6374                                 continue;
6375 
6376                         case SCF_ERROR_NOT_SET:
6377                         case SCF_ERROR_PERMISSION_DENIED:
6378                                 bad_error("scf_property_get_value",
6379                                     scf_error());
6380                         }
6381                 }
6382 
6383                 r = scf_value_get_integer(val, &action_ts[i]);
6384                 assert(r == 0);
6385         }
6386 
6387         a = ADMIN_EVENT_MAINT_ON_IMMEDIATE;
6388         if (action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ||
6389             action_ts[ADMIN_EVENT_MAINT_ON]) {
6390                 a = action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ?
6391                     ADMIN_EVENT_MAINT_ON_IMMEDIATE : ADMIN_EVENT_MAINT_ON;
6392 
6393                 vertex_send_event(vertex, admin_events[a]);
6394                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6395                 switch (r) {
6396                 case 0:
6397                 case EACCES:
6398                         break;
6399 
6400                 case ECONNABORTED:
6401                         ret = ECONNABORTED;
6402                         goto out;
6403 
6404                 case EPERM:
6405                         uu_die("Insufficient privilege.\n");
6406                         /* NOTREACHED */
6407 
6408                 default:
6409                         bad_error("libscf_unset_action", r);
6410                 }
6411         }
6412 
6413         while ((a = next_action(action_ts, NACTIONS)) != -1) {
6414                 log_framework(LOG_DEBUG,
6415                     "Graph: processing %s action for %s.\n", admin_actions[a],
6416                     inst_name);
6417 
6418                 if (a == ADMIN_EVENT_REFRESH) {
6419                         r = dgraph_refresh_instance(vertex, inst);
6420                         switch (r) {
6421                         case 0:
6422                         case ECANCELED:
6423                         case EINVAL:
6424                         case -1:
6425                                 break;
6426 
6427                         case ECONNABORTED:
6428                                 /* pg & inst are reset now, so just return. */
6429                                 ret = ECONNABORTED;
6430                                 goto out;
6431 
6432                         default:
6433                                 bad_error("dgraph_refresh_instance", r);
6434                         }
6435                 }
6436 
6437                 vertex_send_event(vertex, admin_events[a]);
6438 
6439                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6440                 switch (r) {
6441                 case 0:
6442                 case EACCES:
6443                         break;
6444 
6445                 case ECONNABORTED:
6446                         ret = ECONNABORTED;
6447                         goto out;
6448 
6449                 case EPERM:
6450                         uu_die("Insufficient privilege.\n");
6451                         /* NOTREACHED */
6452 
6453                 default:
6454                         bad_error("libscf_unset_action", r);
6455                 }
6456 
6457                 action_ts[a] = 0;
6458         }
6459 
6460 out:
6461         MUTEX_UNLOCK(&dgraph_lock);
6462 
6463         scf_property_destroy(prop);
6464         scf_value_destroy(val);
6465         startd_free(inst_name, max_scf_fmri_size);
6466         return (ret);
6467 }
6468 
6469 /*
6470  * inst and pg_name are scratch space, and are unset on entry.
6471  * Returns
6472  *   0 - success
6473  *   ECONNRESET - success, but repository handle rebound
6474  *   ECONNABORTED - repository connection broken
6475  */
6476 static int
6477 process_pg_event(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst,
6478     char *pg_name)
6479 {
6480         int r;
6481         scf_property_t *prop;
6482         scf_value_t *val;
6483         char *fmri;
6484         boolean_t rebound = B_FALSE, rebind_inst = B_FALSE;
6485 
6486         if (scf_pg_get_name(pg, pg_name, max_scf_value_size) < 0) {
6487                 switch (scf_error()) {
6488                 case SCF_ERROR_CONNECTION_BROKEN:
6489                 default:
6490                         return (ECONNABORTED);
6491 
6492                 case SCF_ERROR_DELETED:
6493                         return (0);
6494 
6495                 case SCF_ERROR_NOT_SET:
6496                         bad_error("scf_pg_get_name", scf_error());
6497                 }
6498         }
6499 
6500         if (strcmp(pg_name, SCF_PG_GENERAL) == 0 ||
6501             strcmp(pg_name, SCF_PG_GENERAL_OVR) == 0) {
6502                 r = dgraph_update_general(pg);
6503                 switch (r) {
6504                 case 0:
6505                 case ENOTSUP:
6506                 case ECANCELED:
6507                         return (0);
6508 
6509                 case ECONNABORTED:
6510                         return (ECONNABORTED);
6511 
6512                 case -1:
6513                         /* Error should have been logged. */
6514                         return (0);
6515 
6516                 default:
6517                         bad_error("dgraph_update_general", r);
6518                 }
6519         } else if (strcmp(pg_name, SCF_PG_RESTARTER_ACTIONS) == 0) {
6520                 if (scf_pg_get_parent_instance(pg, inst) != 0) {
6521                         switch (scf_error()) {
6522                         case SCF_ERROR_CONNECTION_BROKEN:
6523                                 return (ECONNABORTED);
6524 
6525                         case SCF_ERROR_DELETED:
6526                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6527                                 /* Ignore commands on services. */
6528                                 return (0);
6529 
6530                         case SCF_ERROR_NOT_BOUND:
6531                         case SCF_ERROR_HANDLE_MISMATCH:
6532                         case SCF_ERROR_NOT_SET:
6533                         default:
6534                                 bad_error("scf_pg_get_parent_instance",
6535                                     scf_error());
6536                         }
6537                 }
6538 
6539                 return (process_actions(h, pg, inst));
6540         }
6541 
6542         if (strcmp(pg_name, SCF_PG_OPTIONS) != 0 &&
6543             strcmp(pg_name, SCF_PG_OPTIONS_OVR) != 0)
6544                 return (0);
6545 
6546         /*
6547          * We only care about the options[_ovr] property groups of our own
6548          * instance, so get the fmri and compare.  Plus, once we know it's
6549          * correct, if the repository connection is broken we know exactly what
6550          * property group we were operating on, and can look it up again.
6551          */
6552         if (scf_pg_get_parent_instance(pg, inst) != 0) {
6553                 switch (scf_error()) {
6554                 case SCF_ERROR_CONNECTION_BROKEN:
6555                         return (ECONNABORTED);
6556 
6557                 case SCF_ERROR_DELETED:
6558                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6559                         return (0);
6560 
6561                 case SCF_ERROR_HANDLE_MISMATCH:
6562                 case SCF_ERROR_NOT_BOUND:
6563                 case SCF_ERROR_NOT_SET:
6564                 default:
6565                         bad_error("scf_pg_get_parent_instance",
6566                             scf_error());
6567                 }
6568         }
6569 
6570         switch (r = libscf_instance_get_fmri(inst, &fmri)) {
6571         case 0:
6572                 break;
6573 
6574         case ECONNABORTED:
6575                 return (ECONNABORTED);
6576 
6577         case ECANCELED:
6578                 return (0);
6579 
6580         default:
6581                 bad_error("libscf_instance_get_fmri", r);
6582         }
6583 
6584         if (strcmp(fmri, SCF_SERVICE_STARTD) != 0) {
6585                 startd_free(fmri, max_scf_fmri_size);
6586                 return (0);
6587         }
6588 
6589         /*
6590          * update the information events flag
6591          */
6592         if (strcmp(pg_name, SCF_PG_OPTIONS) == 0)
6593                 info_events_all = libscf_get_info_events_all(pg);
6594 
6595         prop = safe_scf_property_create(h);
6596         val = safe_scf_value_create(h);
6597 
6598         if (strcmp(pg_name, SCF_PG_OPTIONS_OVR) == 0) {
6599                 /* See if we need to set the runlevel. */
6600                 /* CONSTCOND */
6601                 if (0) {
6602 rebind_pg:
6603                         libscf_handle_rebind(h);
6604                         rebound = B_TRUE;
6605 
6606                         r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6607                         switch (r) {
6608                         case 0:
6609                                 break;
6610 
6611                         case ECONNABORTED:
6612                                 goto rebind_pg;
6613 
6614                         case ENOENT:
6615                                 goto out;
6616 
6617                         case EINVAL:
6618                         case ENOTSUP:
6619                                 bad_error("libscf_lookup_instance", r);
6620                         }
6621 
6622                         if (scf_instance_get_pg(inst, pg_name, pg) != 0) {
6623                                 switch (scf_error()) {
6624                                 case SCF_ERROR_DELETED:
6625                                 case SCF_ERROR_NOT_FOUND:
6626                                         goto out;
6627 
6628                                 case SCF_ERROR_CONNECTION_BROKEN:
6629                                         goto rebind_pg;
6630 
6631                                 case SCF_ERROR_HANDLE_MISMATCH:
6632                                 case SCF_ERROR_NOT_BOUND:
6633                                 case SCF_ERROR_NOT_SET:
6634                                 case SCF_ERROR_INVALID_ARGUMENT:
6635                                 default:
6636                                         bad_error("scf_instance_get_pg",
6637                                             scf_error());
6638                                 }
6639                         }
6640                 }
6641 
6642                 if (scf_pg_get_property(pg, "runlevel", prop) == 0) {
6643                         r = dgraph_set_runlevel(pg, prop);
6644                         switch (r) {
6645                         case ECONNRESET:
6646                                 rebound = B_TRUE;
6647                                 rebind_inst = B_TRUE;
6648                                 /* FALLTHROUGH */
6649 
6650                         case 0:
6651                                 break;
6652 
6653                         case ECONNABORTED:
6654                                 goto rebind_pg;
6655 
6656                         case ECANCELED:
6657                                 goto out;
6658 
6659                         default:
6660                                 bad_error("dgraph_set_runlevel", r);
6661                         }
6662                 } else {
6663                         switch (scf_error()) {
6664                         case SCF_ERROR_CONNECTION_BROKEN:
6665                         default:
6666                                 goto rebind_pg;
6667 
6668                         case SCF_ERROR_DELETED:
6669                                 goto out;
6670 
6671                         case SCF_ERROR_NOT_FOUND:
6672                                 break;
6673 
6674                         case SCF_ERROR_INVALID_ARGUMENT:
6675                         case SCF_ERROR_HANDLE_MISMATCH:
6676                         case SCF_ERROR_NOT_BOUND:
6677                         case SCF_ERROR_NOT_SET:
6678                                 bad_error("scf_pg_get_property", scf_error());
6679                         }
6680                 }
6681         }
6682 
6683         if (rebind_inst) {
6684 lookup_inst:
6685                 r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6686                 switch (r) {
6687                 case 0:
6688                         break;
6689 
6690                 case ECONNABORTED:
6691                         libscf_handle_rebind(h);
6692                         rebound = B_TRUE;
6693                         goto lookup_inst;
6694 
6695                 case ENOENT:
6696                         goto out;
6697 
6698                 case EINVAL:
6699                 case ENOTSUP:
6700                         bad_error("libscf_lookup_instance", r);
6701                 }
6702         }
6703 
6704         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6705         switch (r) {
6706         case 0:
6707                 break;
6708 
6709         case ECONNABORTED:
6710                 libscf_handle_rebind(h);
6711                 rebound = B_TRUE;
6712                 goto lookup_inst;
6713 
6714         case EINVAL:
6715                 log_error(LOG_NOTICE,
6716                     "%s/%s property of %s is misconfigured.\n", pg_name,
6717                     SCF_PROPERTY_MILESTONE, SCF_SERVICE_STARTD);
6718                 /* FALLTHROUGH */
6719 
6720         case ECANCELED:
6721         case ENOENT:
6722                 (void) strcpy(fmri, "all");
6723                 break;
6724 
6725         default:
6726                 bad_error("libscf_get_milestone", r);
6727         }
6728 
6729         r = dgraph_set_milestone(fmri, h, B_FALSE);
6730         switch (r) {
6731         case 0:
6732         case ECONNRESET:
6733         case EALREADY:
6734                 break;
6735 
6736         case EINVAL:
6737                 log_error(LOG_WARNING, "Milestone %s is invalid.\n", fmri);
6738                 break;
6739 
6740         case ENOENT:
6741                 log_error(LOG_WARNING, "Milestone %s does not exist.\n", fmri);
6742                 break;
6743 
6744         default:
6745                 bad_error("dgraph_set_milestone", r);
6746         }
6747 
6748 out:
6749         startd_free(fmri, max_scf_fmri_size);
6750         scf_value_destroy(val);
6751         scf_property_destroy(prop);
6752 
6753         return (rebound ? ECONNRESET : 0);
6754 }
6755 
6756 /*
6757  * process_delete() deletes an instance from the dgraph if 'fmri' is an
6758  * instance fmri or if 'fmri' matches the 'general' property group of an
6759  * instance (or the 'general/enabled' property).
6760  *
6761  * 'fmri' may be overwritten and cannot be trusted on return by the caller.
6762  */
6763 static void
6764 process_delete(char *fmri, scf_handle_t *h)
6765 {
6766         char *lfmri, *end_inst_fmri;
6767         const char *inst_name = NULL;
6768         const char *pg_name = NULL;
6769         const char *prop_name = NULL;
6770 
6771         lfmri = safe_strdup(fmri);
6772 
6773         /* Determine if the FMRI is a property group or instance */
6774         if (scf_parse_svc_fmri(lfmri, NULL, NULL, &inst_name, &pg_name,
6775             &prop_name) != SCF_SUCCESS) {
6776                 log_error(LOG_WARNING,
6777                     "Received invalid FMRI \"%s\" from repository server.\n",
6778                     fmri);
6779         } else if (inst_name != NULL && pg_name == NULL) {
6780                 (void) dgraph_remove_instance(fmri, h);
6781         } else if (inst_name != NULL && pg_name != NULL) {
6782                 /*
6783                  * If we're deleting the 'general' property group or
6784                  * 'general/enabled' property then the whole instance
6785                  * must be removed from the dgraph.
6786                  */
6787                 if (strcmp(pg_name, SCF_PG_GENERAL) != 0) {
6788                         free(lfmri);
6789                         return;
6790                 }
6791 
6792                 if (prop_name != NULL &&
6793                     strcmp(prop_name, SCF_PROPERTY_ENABLED) != 0) {
6794                         free(lfmri);
6795                         return;
6796                 }
6797 
6798                 /*
6799                  * Because the instance has already been deleted from the
6800                  * repository, we cannot use any scf_ functions to retrieve
6801                  * the instance FMRI however we can easily reconstruct it
6802                  * manually.
6803                  */
6804                 end_inst_fmri = strstr(fmri, SCF_FMRI_PROPERTYGRP_PREFIX);
6805                 if (end_inst_fmri == NULL)
6806                         bad_error("process_delete", 0);
6807 
6808                 end_inst_fmri[0] = '\0';
6809 
6810                 (void) dgraph_remove_instance(fmri, h);
6811         }
6812 
6813         free(lfmri);
6814 }
6815 
6816 /*ARGSUSED*/
6817 void *
6818 repository_event_thread(void *unused)
6819 {
6820         scf_handle_t *h;
6821         scf_propertygroup_t *pg;
6822         scf_instance_t *inst;
6823         char *fmri = startd_alloc(max_scf_fmri_size);
6824         char *pg_name = startd_alloc(max_scf_value_size);
6825         int r;
6826 
6827         h = libscf_handle_create_bound_loop();
6828 
6829         pg = safe_scf_pg_create(h);
6830         inst = safe_scf_instance_create(h);
6831 
6832 retry:
6833         if (_scf_notify_add_pgtype(h, SCF_GROUP_FRAMEWORK) != SCF_SUCCESS) {
6834                 if (scf_error() == SCF_ERROR_CONNECTION_BROKEN) {
6835                         libscf_handle_rebind(h);
6836                 } else {
6837                         log_error(LOG_WARNING,
6838                             "Couldn't set up repository notification "
6839                             "for property group type %s: %s\n",
6840                             SCF_GROUP_FRAMEWORK, scf_strerror(scf_error()));
6841 
6842                         (void) sleep(1);
6843                 }
6844 
6845                 goto retry;
6846         }
6847 
6848         /*CONSTCOND*/
6849         while (1) {
6850                 ssize_t res;
6851 
6852                 /* Note: fmri is only set on delete events. */
6853                 res = _scf_notify_wait(pg, fmri, max_scf_fmri_size);
6854                 if (res < 0) {
6855                         libscf_handle_rebind(h);
6856                         goto retry;
6857                 } else if (res == 0) {
6858                         /*
6859                          * property group modified.  inst and pg_name are
6860                          * pre-allocated scratch space.
6861                          */
6862                         if (scf_pg_update(pg) < 0) {
6863                                 switch (scf_error()) {
6864                                 case SCF_ERROR_DELETED:
6865                                         continue;
6866 
6867                                 case SCF_ERROR_CONNECTION_BROKEN:
6868                                         log_error(LOG_WARNING,
6869                                             "Lost repository event due to "
6870                                             "disconnection.\n");
6871                                         libscf_handle_rebind(h);
6872                                         goto retry;
6873 
6874                                 case SCF_ERROR_NOT_BOUND:
6875                                 case SCF_ERROR_NOT_SET:
6876                                 default:
6877                                         bad_error("scf_pg_update", scf_error());
6878                                 }
6879                         }
6880 
6881                         r = process_pg_event(h, pg, inst, pg_name);
6882                         switch (r) {
6883                         case 0:
6884                                 break;
6885 
6886                         case ECONNABORTED:
6887                                 log_error(LOG_WARNING, "Lost repository event "
6888                                     "due to disconnection.\n");
6889                                 libscf_handle_rebind(h);
6890                                 /* FALLTHROUGH */
6891 
6892                         case ECONNRESET:
6893                                 goto retry;
6894 
6895                         default:
6896                                 bad_error("process_pg_event", r);
6897                         }
6898                 } else {
6899                         /*
6900                          * Service, instance, or pg deleted.
6901                          * Don't trust fmri on return.
6902                          */
6903                         process_delete(fmri, h);
6904                 }
6905         }
6906 
6907         /*NOTREACHED*/
6908         return (NULL);
6909 }
6910 
6911 void
6912 graph_engine_start()
6913 {
6914         int err;
6915 
6916         (void) startd_thread_create(graph_thread, NULL);
6917 
6918         MUTEX_LOCK(&dgraph_lock);
6919         while (!initial_milestone_set) {
6920                 err = pthread_cond_wait(&initial_milestone_cv, &dgraph_lock);
6921                 assert(err == 0);
6922         }
6923         MUTEX_UNLOCK(&dgraph_lock);
6924 
6925         (void) startd_thread_create(repository_event_thread, NULL);
6926         (void) startd_thread_create(graph_event_thread, NULL);
6927 }