7387 SMF is fast and loose with require_any/service dependencies
Reviewed by: Dan McDonald <danmcd@omniti.com>

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
  25  * Copyright 2016 RackTop Systems.
  26  */
  27 
  28 /*
  29  * graph.c - master restarter graph engine
  30  *
  31  *   The graph engine keeps a dependency graph of all service instances on the
  32  *   system, as recorded in the repository.  It decides when services should
  33  *   be brought up or down based on service states and dependencies and sends
  34  *   commands to restarters to effect any changes.  It also executes
  35  *   administrator commands sent by svcadm via the repository.
  36  *
  37  *   The graph is stored in uu_list_t *dgraph and its vertices are
  38  *   graph_vertex_t's, each of which has a name and an integer id unique to
  39  *   its name (see dict.c).  A vertex's type attribute designates the type
  40  *   of object it represents: GVT_INST for service instances, GVT_SVC for
  41  *   service objects (since service instances may depend on another service,
  42  *   rather than service instance), GVT_FILE for files (which services may
  43  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
  44  *   vertices are necessary because dependency lists may have particular
  45  *   grouping types (require any, require all, optional, or exclude) and
  46  *   event-propagation characteristics.
  47  *
  48  *   The initial graph is built by libscf_populate_graph() invoking
  49  *   dgraph_add_instance() for each instance in the repository.  The function
  50  *   adds a GVT_SVC vertex for the service if one does not already exist, adds
  51  *   a GVT_INST vertex named by the FMRI of the instance, and sets up the edges.
  52  *   The resulting web of vertices & edges associated with an instance's vertex
  53  *   includes
  54  *
  55  *     - an edge from the GVT_SVC vertex for the instance's service
  56  *
  57  *     - an edge to the GVT_INST vertex of the instance's resarter, if its
  58  *       restarter is not svc.startd
  59  *
  60  *     - edges from other GVT_INST vertices if the instance is a restarter
  61  *
  62  *     - for each dependency property group in the instance's "running"
  63  *       snapshot, an edge to a GVT_GROUP vertex named by the FMRI of the
  64  *       instance and the name of the property group
  65  *
  66  *     - for each value of the "entities" property in each dependency property
  67  *       group, an edge from the corresponding GVT_GROUP vertex to a
  68  *       GVT_INST, GVT_SVC, or GVT_FILE vertex
  69  *
  70  *     - edges from GVT_GROUP vertices for each dependent instance
  71  *
  72  *   After the edges are set up the vertex's GV_CONFIGURED flag is set.  If
  73  *   there are problems, or if a service is mentioned in a dependency but does
  74  *   not exist in the repository, the GV_CONFIGURED flag will be clear.
  75  *
  76  *   The graph and all of its vertices are protected by the dgraph_lock mutex.
  77  *   See restarter.c for more information.
  78  *
  79  *   The properties of an instance fall into two classes: immediate and
  80  *   snapshotted.  Immediate properties should have an immediate effect when
  81  *   changed.  Snapshotted properties should be read from a snapshot, so they
  82  *   only change when the snapshot changes.  The immediate properties used by
  83  *   the graph engine are general/enabled, general/restarter, and the properties
  84  *   in the restarter_actions property group.  Since they are immediate, they
  85  *   are not read out of a snapshot.  The snapshotted properties used by the
  86  *   graph engine are those in the property groups with type "dependency" and
  87  *   are read out of the "running" snapshot.  The "running" snapshot is created
  88  *   by the the graph engine as soon as possible, and it is updated, along with
  89  *   in-core copies of the data (dependency information for the graph engine) on
  90  *   receipt of the refresh command from svcadm.  In addition, the graph engine
  91  *   updates the "start" snapshot from the "running" snapshot whenever a service
  92  *   comes online.
  93  *
  94  *   When a DISABLE event is requested by the administrator, svc.startd shutdown
  95  *   the dependents first before shutting down the requested service.
  96  *   In graph_enable_by_vertex, we create a subtree that contains the dependent
  97  *   vertices by marking those vertices with the GV_TOOFFLINE flag. And we mark
  98  *   the vertex to disable with the GV_TODISABLE flag. Once the tree is created,
  99  *   we send the _ADMIN_DISABLE event to the leaves. The leaves will then
 100  *   transition from STATE_ONLINE/STATE_DEGRADED to STATE_OFFLINE/STATE_MAINT.
 101  *   In gt_enter_offline and gt_enter_maint if the vertex was in a subtree then
 102  *   we clear the GV_TOOFFLINE flag and walk the dependencies to offline the new
 103  *   exposed leaves. We do the same until we reach the last leaf (the one with
 104  *   the GV_TODISABLE flag). If the vertex to disable is also part of a larger
 105  *   subtree (eg. multiple DISABLE events on vertices in the same subtree) then
 106  *   once the first vertex is disabled (GV_TODISABLE flag is removed), we
 107  *   continue to propagate the offline event to the vertex's dependencies.
 108  *
 109  *
 110  * SMF state transition notifications
 111  *
 112  *   When an instance of a service managed by SMF changes state, svc.startd may
 113  *   publish a GPEC sysevent. All transitions to or from maintenance, a
 114  *   transition cause by a hardware error will generate an event.
 115  *   Other transitions will generate an event if there exist notification
 116  *   parameter for that transition. Notification parameters are stored in the
 117  *   SMF repository for the service/instance they refer to. System-wide
 118  *   notification parameters are stored in the global instance.
 119  *   svc.startd can be told to send events for all SMF state transitions despite
 120  *   of notification parameters by setting options/info_events_all to true in
 121  *   restarter:default
 122  *
 123  *   The set of transitions that generate events is cached in the
 124  *   dgraph_vertex_t gv_stn_tset for service/instance and in the global
 125  *   stn_global for the system-wide set. They are re-read when instances are
 126  *   refreshed.
 127  *
 128  *   The GPEC events published by svc.startd are consumed by fmd(1M). After
 129  *   processing these events, fmd(1M) publishes the processed events to
 130  *   notification agents. The notification agents read the notification
 131  *   parameters from the SMF repository through libscf(3LIB) interfaces and send
 132  *   the notification, or not, based on those parameters.
 133  *
 134  *   Subscription and publishing to the GPEC channels is done with the
 135  *   libfmevent(3LIB) wrappers fmev_[r]publish_*() and
 136  *   fmev_shdl_(un)subscribe().
 137  *
 138  */
 139 
 140 #include <sys/uadmin.h>
 141 #include <sys/wait.h>
 142 
 143 #include <assert.h>
 144 #include <errno.h>
 145 #include <fcntl.h>
 146 #include <fm/libfmevent.h>
 147 #include <libscf.h>
 148 #include <libscf_priv.h>
 149 #include <librestart.h>
 150 #include <libuutil.h>
 151 #include <locale.h>
 152 #include <poll.h>
 153 #include <pthread.h>
 154 #include <signal.h>
 155 #include <stddef.h>
 156 #include <stdio.h>
 157 #include <stdlib.h>
 158 #include <string.h>
 159 #include <strings.h>
 160 #include <sys/statvfs.h>
 161 #include <sys/uadmin.h>
 162 #include <zone.h>
 163 #if defined(__i386)
 164 #include <libgrubmgmt.h>
 165 #endif  /* __i386 */
 166 
 167 #include "startd.h"
 168 #include "protocol.h"
 169 
 170 
 171 #define MILESTONE_NONE  ((graph_vertex_t *)1)
 172 
 173 #define CONSOLE_LOGIN_FMRI      "svc:/system/console-login:default"
 174 #define FS_MINIMAL_FMRI         "svc:/system/filesystem/minimal:default"
 175 
 176 #define VERTEX_REMOVED  0       /* vertex has been freed  */
 177 #define VERTEX_INUSE    1       /* vertex is still in use */
 178 
 179 #define IS_ENABLED(v) ((v)->gv_flags & (GV_ENABLED | GV_ENBLD_NOOVR))
 180 
 181 /*
 182  * stn_global holds the tset for the system wide notification parameters.
 183  * It is updated on refresh of svc:/system/svc/global:default
 184  *
 185  * There are two assumptions that relax the need for a mutex:
 186  *     1. 32-bit value assignments are atomic
 187  *     2. Its value is consumed only in one point at
 188  *     dgraph_state_transition_notify(). There are no test and set races.
 189  *
 190  *     If either assumption is broken, we'll need a mutex to synchronize
 191  *     access to stn_global
 192  */
 193 int32_t stn_global;
 194 /*
 195  * info_events_all holds a flag to override notification parameters and send
 196  * Information events for all state transitions.
 197  * same about the need of a mutex here.
 198  */
 199 int info_events_all;
 200 
 201 /*
 202  * Services in these states are not considered 'down' by the
 203  * milestone/shutdown code.
 204  */
 205 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
 206         (state) == RESTARTER_STATE_DEGRADED || \
 207         (state) == RESTARTER_STATE_OFFLINE)
 208 
 209 #define is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
 210         ((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \
 211         (v->gv_depgroup == DEPGRP_OPTIONAL_ALL) || \
 212         (v->gv_restart < RERR_RESTART)))
 213 
 214 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
 215 static uu_list_t *dgraph;
 216 static pthread_mutex_t dgraph_lock;
 217 
 218 /*
 219  * milestone indicates the current subgraph.  When NULL, it is the entire
 220  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
 221  * services on which the target vertex depends.
 222  */
 223 static graph_vertex_t *milestone = NULL;
 224 static boolean_t initial_milestone_set = B_FALSE;
 225 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
 226 
 227 /* protected by dgraph_lock */
 228 static boolean_t sulogin_thread_running = B_FALSE;
 229 static boolean_t sulogin_running = B_FALSE;
 230 static boolean_t console_login_ready = B_FALSE;
 231 
 232 /* Number of services to come down to complete milestone transition. */
 233 static uint_t non_subgraph_svcs;
 234 
 235 /*
 236  * These variables indicate what should be done when we reach the milestone
 237  * target milestone, i.e., when non_subgraph_svcs == 0.  They are acted upon in
 238  * dgraph_set_instance_state().
 239  */
 240 static int halting = -1;
 241 static boolean_t go_single_user_mode = B_FALSE;
 242 static boolean_t go_to_level1 = B_FALSE;
 243 
 244 /*
 245  * Tracks when we started halting.
 246  */
 247 static time_t halting_time = 0;
 248 
 249 /*
 250  * This tracks the legacy runlevel to ensure we signal init and manage
 251  * utmpx entries correctly.
 252  */
 253 static char current_runlevel = '\0';
 254 
 255 /* Number of single user threads currently running */
 256 static pthread_mutex_t single_user_thread_lock;
 257 static int single_user_thread_count = 0;
 258 
 259 /* Statistics for dependency cycle-checking */
 260 static u_longlong_t dep_inserts = 0;
 261 static u_longlong_t dep_cycle_ns = 0;
 262 static u_longlong_t dep_insert_ns = 0;
 263 
 264 
 265 static const char * const emsg_invalid_restarter =
 266         "Transitioning %s to maintenance, restarter FMRI %s is invalid "
 267         "(see 'svcs -xv' for details).\n";
 268 static const char * const console_login_fmri = CONSOLE_LOGIN_FMRI;
 269 static const char * const single_user_fmri = SCF_MILESTONE_SINGLE_USER;
 270 static const char * const multi_user_fmri = SCF_MILESTONE_MULTI_USER;
 271 static const char * const multi_user_svr_fmri = SCF_MILESTONE_MULTI_USER_SERVER;
 272 
 273 
 274 /*
 275  * These services define the system being "up".  If none of them can come
 276  * online, then we will run sulogin on the console.  Note that the install ones
 277  * are for the miniroot and when installing CDs after the first.  can_come_up()
 278  * does the decision making, and an sulogin_thread() runs sulogin, which can be
 279  * started by dgraph_set_instance_state() or single_user_thread().
 280  *
 281  * NOTE: can_come_up() relies on SCF_MILESTONE_SINGLE_USER being the first
 282  * entry, which is only used when booting_to_single_user (boot -s) is set.
 283  * This is because when doing a "boot -s", sulogin is started from specials.c
 284  * after milestone/single-user comes online, for backwards compatibility.
 285  * In this case, SCF_MILESTONE_SINGLE_USER needs to be part of up_svcs
 286  * to ensure sulogin will be spawned if milestone/single-user cannot be reached.
 287  */
 288 static const char * const up_svcs[] = {
 289         SCF_MILESTONE_SINGLE_USER,
 290         CONSOLE_LOGIN_FMRI,
 291         "svc:/system/install-setup:default",
 292         "svc:/system/install:default",
 293         NULL
 294 };
 295 
 296 /* This array must have an element for each non-NULL element of up_svcs[]. */
 297 static graph_vertex_t *up_svcs_p[] = { NULL, NULL, NULL, NULL };
 298 
 299 /* These are for seed repository magic.  See can_come_up(). */
 300 static const char * const manifest_import = SCF_INSTANCE_MI;
 301 static graph_vertex_t *manifest_import_p = NULL;
 302 
 303 
 304 static char target_milestone_as_runlevel(void);
 305 static void graph_runlevel_changed(char rl, int online);
 306 static int dgraph_set_milestone(const char *, scf_handle_t *, boolean_t);
 307 static boolean_t should_be_in_subgraph(graph_vertex_t *v);
 308 static int mark_subtree(graph_edge_t *, void *);
 309 static boolean_t insubtree_dependents_down(graph_vertex_t *);
 310 
 311 /*
 312  * graph_vertex_compare()
 313  *      This function can compare either int *id or * graph_vertex_t *gv
 314  *      values, as the vertex id is always the first element of a
 315  *      graph_vertex structure.
 316  */
 317 /* ARGSUSED */
 318 static int
 319 graph_vertex_compare(const void *lc_arg, const void *rc_arg, void *private)
 320 {
 321         int lc_id = ((const graph_vertex_t *)lc_arg)->gv_id;
 322         int rc_id = *(int *)rc_arg;
 323 
 324         if (lc_id > rc_id)
 325                 return (1);
 326         if (lc_id < rc_id)
 327                 return (-1);
 328         return (0);
 329 }
 330 
 331 void
 332 graph_init()
 333 {
 334         graph_edge_pool = startd_list_pool_create("graph_edges",
 335             sizeof (graph_edge_t), offsetof(graph_edge_t, ge_link), NULL,
 336             UU_LIST_POOL_DEBUG);
 337         assert(graph_edge_pool != NULL);
 338 
 339         graph_vertex_pool = startd_list_pool_create("graph_vertices",
 340             sizeof (graph_vertex_t), offsetof(graph_vertex_t, gv_link),
 341             graph_vertex_compare, UU_LIST_POOL_DEBUG);
 342         assert(graph_vertex_pool != NULL);
 343 
 344         (void) pthread_mutex_init(&dgraph_lock, &mutex_attrs);
 345         (void) pthread_mutex_init(&single_user_thread_lock, &mutex_attrs);
 346         dgraph = startd_list_create(graph_vertex_pool, NULL, UU_LIST_SORTED);
 347         assert(dgraph != NULL);
 348 
 349         if (!st->st_initial)
 350                 current_runlevel = utmpx_get_runlevel();
 351 
 352         log_framework(LOG_DEBUG, "Initialized graph\n");
 353 }
 354 
 355 static graph_vertex_t *
 356 vertex_get_by_name(const char *name)
 357 {
 358         int id;
 359 
 360         assert(MUTEX_HELD(&dgraph_lock));
 361 
 362         id = dict_lookup_byname(name);
 363         if (id == -1)
 364                 return (NULL);
 365 
 366         return (uu_list_find(dgraph, &id, NULL, NULL));
 367 }
 368 
 369 static graph_vertex_t *
 370 vertex_get_by_id(int id)
 371 {
 372         assert(MUTEX_HELD(&dgraph_lock));
 373 
 374         if (id == -1)
 375                 return (NULL);
 376 
 377         return (uu_list_find(dgraph, &id, NULL, NULL));
 378 }
 379 
 380 /*
 381  * Creates a new vertex with the given name, adds it to the graph, and returns
 382  * a pointer to it.  The graph lock must be held by this thread on entry.
 383  */
 384 static graph_vertex_t *
 385 graph_add_vertex(const char *name)
 386 {
 387         int id;
 388         graph_vertex_t *v;
 389         void *p;
 390         uu_list_index_t idx;
 391 
 392         assert(MUTEX_HELD(&dgraph_lock));
 393 
 394         id = dict_insert(name);
 395 
 396         v = startd_zalloc(sizeof (*v));
 397 
 398         v->gv_id = id;
 399 
 400         v->gv_name = startd_alloc(strlen(name) + 1);
 401         (void) strcpy(v->gv_name, name);
 402 
 403         v->gv_dependencies = startd_list_create(graph_edge_pool, v, 0);
 404         v->gv_dependents = startd_list_create(graph_edge_pool, v, 0);
 405 
 406         p = uu_list_find(dgraph, &id, NULL, &idx);
 407         assert(p == NULL);
 408 
 409         uu_list_node_init(v, &v->gv_link, graph_vertex_pool);
 410         uu_list_insert(dgraph, v, idx);
 411 
 412         return (v);
 413 }
 414 
 415 /*
 416  * Removes v from the graph and frees it.  The graph should be locked by this
 417  * thread, and v should have no edges associated with it.
 418  */
 419 static void
 420 graph_remove_vertex(graph_vertex_t *v)
 421 {
 422         assert(MUTEX_HELD(&dgraph_lock));
 423 
 424         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 425         assert(uu_list_numnodes(v->gv_dependents) == 0);
 426         assert(v->gv_refs == 0);
 427 
 428         startd_free(v->gv_name, strlen(v->gv_name) + 1);
 429         uu_list_destroy(v->gv_dependencies);
 430         uu_list_destroy(v->gv_dependents);
 431         uu_list_remove(dgraph, v);
 432 
 433         startd_free(v, sizeof (graph_vertex_t));
 434 }
 435 
 436 static void
 437 graph_add_edge(graph_vertex_t *fv, graph_vertex_t *tv)
 438 {
 439         graph_edge_t *e, *re;
 440         int r;
 441 
 442         assert(MUTEX_HELD(&dgraph_lock));
 443 
 444         e = startd_alloc(sizeof (graph_edge_t));
 445         re = startd_alloc(sizeof (graph_edge_t));
 446 
 447         e->ge_parent = fv;
 448         e->ge_vertex = tv;
 449 
 450         re->ge_parent = tv;
 451         re->ge_vertex = fv;
 452 
 453         uu_list_node_init(e, &e->ge_link, graph_edge_pool);
 454         r = uu_list_insert_before(fv->gv_dependencies, NULL, e);
 455         assert(r == 0);
 456 
 457         uu_list_node_init(re, &re->ge_link, graph_edge_pool);
 458         r = uu_list_insert_before(tv->gv_dependents, NULL, re);
 459         assert(r == 0);
 460 }
 461 
 462 static void
 463 graph_remove_edge(graph_vertex_t *v, graph_vertex_t *dv)
 464 {
 465         graph_edge_t *e;
 466 
 467         for (e = uu_list_first(v->gv_dependencies);
 468             e != NULL;
 469             e = uu_list_next(v->gv_dependencies, e)) {
 470                 if (e->ge_vertex == dv) {
 471                         uu_list_remove(v->gv_dependencies, e);
 472                         startd_free(e, sizeof (graph_edge_t));
 473                         break;
 474                 }
 475         }
 476 
 477         for (e = uu_list_first(dv->gv_dependents);
 478             e != NULL;
 479             e = uu_list_next(dv->gv_dependents, e)) {
 480                 if (e->ge_vertex == v) {
 481                         uu_list_remove(dv->gv_dependents, e);
 482                         startd_free(e, sizeof (graph_edge_t));
 483                         break;
 484                 }
 485         }
 486 }
 487 
 488 static void
 489 remove_inst_vertex(graph_vertex_t *v)
 490 {
 491         graph_edge_t *e;
 492         graph_vertex_t *sv;
 493         int i;
 494 
 495         assert(MUTEX_HELD(&dgraph_lock));
 496         assert(uu_list_numnodes(v->gv_dependents) == 1);
 497         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 498         assert(v->gv_refs == 0);
 499         assert((v->gv_flags & GV_CONFIGURED) == 0);
 500 
 501         e = uu_list_first(v->gv_dependents);
 502         sv = e->ge_vertex;
 503         graph_remove_edge(sv, v);
 504 
 505         for (i = 0; up_svcs[i] != NULL; ++i) {
 506                 if (up_svcs_p[i] == v)
 507                         up_svcs_p[i] = NULL;
 508         }
 509 
 510         if (manifest_import_p == v)
 511                 manifest_import_p = NULL;
 512 
 513         graph_remove_vertex(v);
 514 
 515         if (uu_list_numnodes(sv->gv_dependencies) == 0 &&
 516             uu_list_numnodes(sv->gv_dependents) == 0 &&
 517             sv->gv_refs == 0)
 518                 graph_remove_vertex(sv);
 519 }
 520 
 521 static void
 522 graph_walk_dependents(graph_vertex_t *v, void (*func)(graph_vertex_t *, void *),
 523     void *arg)
 524 {
 525         graph_edge_t *e;
 526 
 527         for (e = uu_list_first(v->gv_dependents);
 528             e != NULL;
 529             e = uu_list_next(v->gv_dependents, e))
 530                 func(e->ge_vertex, arg);
 531 }
 532 
 533 static void
 534 graph_walk_dependencies(graph_vertex_t *v, void (*func)(graph_vertex_t *,
 535         void *), void *arg)
 536 {
 537         graph_edge_t *e;
 538 
 539         assert(MUTEX_HELD(&dgraph_lock));
 540 
 541         for (e = uu_list_first(v->gv_dependencies);
 542             e != NULL;
 543             e = uu_list_next(v->gv_dependencies, e)) {
 544 
 545                 func(e->ge_vertex, arg);
 546         }
 547 }
 548 
 549 /*
 550  * Generic graph walking function.
 551  *
 552  * Given a vertex, this function will walk either dependencies
 553  * (WALK_DEPENDENCIES) or dependents (WALK_DEPENDENTS) of a vertex recursively
 554  * for the entire graph.  It will avoid cycles and never visit the same vertex
 555  * twice.
 556  *
 557  * We avoid traversing exclusion dependencies, because they are allowed to
 558  * create cycles in the graph.  When propagating satisfiability, there is no
 559  * need to walk exclusion dependencies because exclude_all_satisfied() doesn't
 560  * test for satisfiability.
 561  *
 562  * The walker takes two callbacks.  The first is called before examining the
 563  * dependents of each vertex.  The second is called on each vertex after
 564  * examining its dependents.  This allows is_path_to() to construct a path only
 565  * after the target vertex has been found.
 566  */
 567 typedef enum {
 568         WALK_DEPENDENTS,
 569         WALK_DEPENDENCIES
 570 } graph_walk_dir_t;
 571 
 572 typedef int (*graph_walk_cb_t)(graph_vertex_t *, void *);
 573 
 574 typedef struct graph_walk_info {
 575         graph_walk_dir_t        gi_dir;
 576         uchar_t                 *gi_visited;    /* vertex bitmap */
 577         int                     (*gi_pre)(graph_vertex_t *, void *);
 578         void                    (*gi_post)(graph_vertex_t *, void *);
 579         void                    *gi_arg;        /* callback arg */
 580         int                     gi_ret;         /* return value */
 581 } graph_walk_info_t;
 582 
 583 static int
 584 graph_walk_recurse(graph_edge_t *e, graph_walk_info_t *gip)
 585 {
 586         uu_list_t *list;
 587         int r;
 588         graph_vertex_t *v = e->ge_vertex;
 589         int i;
 590         uint_t b;
 591 
 592         i = v->gv_id / 8;
 593         b = 1 << (v->gv_id % 8);
 594 
 595         /*
 596          * Check to see if we've visited this vertex already.
 597          */
 598         if (gip->gi_visited[i] & b)
 599                 return (UU_WALK_NEXT);
 600 
 601         gip->gi_visited[i] |= b;
 602 
 603         /*
 604          * Don't follow exclusions.
 605          */
 606         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
 607                 return (UU_WALK_NEXT);
 608 
 609         /*
 610          * Call pre-visit callback.  If this doesn't terminate the walk,
 611          * continue search.
 612          */
 613         if ((gip->gi_ret = gip->gi_pre(v, gip->gi_arg)) == UU_WALK_NEXT) {
 614                 /*
 615                  * Recurse using appropriate list.
 616                  */
 617                 if (gip->gi_dir == WALK_DEPENDENTS)
 618                         list = v->gv_dependents;
 619                 else
 620                         list = v->gv_dependencies;
 621 
 622                 r = uu_list_walk(list, (uu_walk_fn_t *)graph_walk_recurse,
 623                     gip, 0);
 624                 assert(r == 0);
 625         }
 626 
 627         /*
 628          * Callbacks must return either UU_WALK_NEXT or UU_WALK_DONE.
 629          */
 630         assert(gip->gi_ret == UU_WALK_NEXT || gip->gi_ret == UU_WALK_DONE);
 631 
 632         /*
 633          * If given a post-callback, call the function for every vertex.
 634          */
 635         if (gip->gi_post != NULL)
 636                 (void) gip->gi_post(v, gip->gi_arg);
 637 
 638         /*
 639          * Preserve the callback's return value.  If the callback returns
 640          * UU_WALK_DONE, then we propagate that to the caller in order to
 641          * terminate the walk.
 642          */
 643         return (gip->gi_ret);
 644 }
 645 
 646 static void
 647 graph_walk(graph_vertex_t *v, graph_walk_dir_t dir,
 648     int (*pre)(graph_vertex_t *, void *),
 649     void (*post)(graph_vertex_t *, void *), void *arg)
 650 {
 651         graph_walk_info_t gi;
 652         graph_edge_t fake;
 653         size_t sz = dictionary->dict_new_id / 8 + 1;
 654 
 655         gi.gi_visited = startd_zalloc(sz);
 656         gi.gi_pre = pre;
 657         gi.gi_post = post;
 658         gi.gi_arg = arg;
 659         gi.gi_dir = dir;
 660         gi.gi_ret = 0;
 661 
 662         /*
 663          * Fake up an edge for the first iteration
 664          */
 665         fake.ge_vertex = v;
 666         (void) graph_walk_recurse(&fake, &gi);
 667 
 668         startd_free(gi.gi_visited, sz);
 669 }
 670 
 671 typedef struct child_search {
 672         int     id;             /* id of vertex to look for */
 673         uint_t  depth;          /* recursion depth */
 674         /*
 675          * While the vertex is not found, path is NULL.  After the search, if
 676          * the vertex was found then path should point to a -1-terminated
 677          * array of vertex id's which constitute the path to the vertex.
 678          */
 679         int     *path;
 680 } child_search_t;
 681 
 682 static int
 683 child_pre(graph_vertex_t *v, void *arg)
 684 {
 685         child_search_t *cs = arg;
 686 
 687         cs->depth++;
 688 
 689         if (v->gv_id == cs->id) {
 690                 cs->path = startd_alloc((cs->depth + 1) * sizeof (int));
 691                 cs->path[cs->depth] = -1;
 692                 return (UU_WALK_DONE);
 693         }
 694 
 695         return (UU_WALK_NEXT);
 696 }
 697 
 698 static void
 699 child_post(graph_vertex_t *v, void *arg)
 700 {
 701         child_search_t *cs = arg;
 702 
 703         cs->depth--;
 704 
 705         if (cs->path != NULL)
 706                 cs->path[cs->depth] = v->gv_id;
 707 }
 708 
 709 /*
 710  * Look for a path from from to to.  If one exists, returns a pointer to
 711  * a NULL-terminated array of pointers to the vertices along the path.  If
 712  * there is no path, returns NULL.
 713  */
 714 static int *
 715 is_path_to(graph_vertex_t *from, graph_vertex_t *to)
 716 {
 717         child_search_t cs;
 718 
 719         cs.id = to->gv_id;
 720         cs.depth = 0;
 721         cs.path = NULL;
 722 
 723         graph_walk(from, WALK_DEPENDENCIES, child_pre, child_post, &cs);
 724 
 725         return (cs.path);
 726 }
 727 
 728 /*
 729  * Given an array of int's as returned by is_path_to, allocates a string of
 730  * their names joined by newlines.  Returns the size of the allocated buffer
 731  * in *sz and frees path.
 732  */
 733 static void
 734 path_to_str(int *path, char **cpp, size_t *sz)
 735 {
 736         int i;
 737         graph_vertex_t *v;
 738         size_t allocd, new_allocd;
 739         char *new, *name;
 740 
 741         assert(MUTEX_HELD(&dgraph_lock));
 742         assert(path[0] != -1);
 743 
 744         allocd = 1;
 745         *cpp = startd_alloc(1);
 746         (*cpp)[0] = '\0';
 747 
 748         for (i = 0; path[i] != -1; ++i) {
 749                 name = NULL;
 750 
 751                 v = vertex_get_by_id(path[i]);
 752 
 753                 if (v == NULL)
 754                         name = "<deleted>";
 755                 else if (v->gv_type == GVT_INST || v->gv_type == GVT_SVC)
 756                         name = v->gv_name;
 757 
 758                 if (name != NULL) {
 759                         new_allocd = allocd + strlen(name) + 1;
 760                         new = startd_alloc(new_allocd);
 761                         (void) strcpy(new, *cpp);
 762                         (void) strcat(new, name);
 763                         (void) strcat(new, "\n");
 764 
 765                         startd_free(*cpp, allocd);
 766 
 767                         *cpp = new;
 768                         allocd = new_allocd;
 769                 }
 770         }
 771 
 772         startd_free(path, sizeof (int) * (i + 1));
 773 
 774         *sz = allocd;
 775 }
 776 
 777 
 778 /*
 779  * This function along with run_sulogin() implements an exclusion relationship
 780  * between system/console-login and sulogin.  run_sulogin() will fail if
 781  * system/console-login is online, and the graph engine should call
 782  * graph_clogin_start() to bring system/console-login online, which defers the
 783  * start if sulogin is running.
 784  */
 785 static void
 786 graph_clogin_start(graph_vertex_t *v)
 787 {
 788         assert(MUTEX_HELD(&dgraph_lock));
 789 
 790         if (sulogin_running)
 791                 console_login_ready = B_TRUE;
 792         else
 793                 vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 794 }
 795 
 796 static void
 797 graph_su_start(graph_vertex_t *v)
 798 {
 799         /*
 800          * /etc/inittab used to have the initial /sbin/rcS as a 'sysinit'
 801          * entry with a runlevel of 'S', before jumping to the final
 802          * target runlevel (as set in initdefault).  We mimic that legacy
 803          * behavior here.
 804          */
 805         utmpx_set_runlevel('S', '0', B_FALSE);
 806         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 807 }
 808 
 809 static void
 810 graph_post_su_online(void)
 811 {
 812         graph_runlevel_changed('S', 1);
 813 }
 814 
 815 static void
 816 graph_post_su_disable(void)
 817 {
 818         graph_runlevel_changed('S', 0);
 819 }
 820 
 821 static void
 822 graph_post_mu_online(void)
 823 {
 824         graph_runlevel_changed('2', 1);
 825 }
 826 
 827 static void
 828 graph_post_mu_disable(void)
 829 {
 830         graph_runlevel_changed('2', 0);
 831 }
 832 
 833 static void
 834 graph_post_mus_online(void)
 835 {
 836         graph_runlevel_changed('3', 1);
 837 }
 838 
 839 static void
 840 graph_post_mus_disable(void)
 841 {
 842         graph_runlevel_changed('3', 0);
 843 }
 844 
 845 static struct special_vertex_info {
 846         const char      *name;
 847         void            (*start_f)(graph_vertex_t *);
 848         void            (*post_online_f)(void);
 849         void            (*post_disable_f)(void);
 850 } special_vertices[] = {
 851         { CONSOLE_LOGIN_FMRI, graph_clogin_start, NULL, NULL },
 852         { SCF_MILESTONE_SINGLE_USER, graph_su_start,
 853             graph_post_su_online, graph_post_su_disable },
 854         { SCF_MILESTONE_MULTI_USER, NULL,
 855             graph_post_mu_online, graph_post_mu_disable },
 856         { SCF_MILESTONE_MULTI_USER_SERVER, NULL,
 857             graph_post_mus_online, graph_post_mus_disable },
 858         { NULL },
 859 };
 860 
 861 
 862 void
 863 vertex_send_event(graph_vertex_t *v, restarter_event_type_t e)
 864 {
 865         switch (e) {
 866         case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
 867                 assert(v->gv_state == RESTARTER_STATE_UNINIT);
 868 
 869                 MUTEX_LOCK(&st->st_load_lock);
 870                 st->st_load_instances++;
 871                 MUTEX_UNLOCK(&st->st_load_lock);
 872                 break;
 873 
 874         case RESTARTER_EVENT_TYPE_ENABLE:
 875                 log_framework(LOG_DEBUG, "Enabling %s.\n", v->gv_name);
 876                 assert(v->gv_state == RESTARTER_STATE_UNINIT ||
 877                     v->gv_state == RESTARTER_STATE_DISABLED ||
 878                     v->gv_state == RESTARTER_STATE_MAINT);
 879                 break;
 880 
 881         case RESTARTER_EVENT_TYPE_DISABLE:
 882         case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
 883                 log_framework(LOG_DEBUG, "Disabling %s.\n", v->gv_name);
 884                 assert(v->gv_state != RESTARTER_STATE_DISABLED);
 885                 break;
 886 
 887         case RESTARTER_EVENT_TYPE_STOP_RESET:
 888         case RESTARTER_EVENT_TYPE_STOP:
 889                 log_framework(LOG_DEBUG, "Stopping %s.\n", v->gv_name);
 890                 assert(v->gv_state == RESTARTER_STATE_DEGRADED ||
 891                     v->gv_state == RESTARTER_STATE_ONLINE);
 892                 break;
 893 
 894         case RESTARTER_EVENT_TYPE_START:
 895                 log_framework(LOG_DEBUG, "Starting %s.\n", v->gv_name);
 896                 assert(v->gv_state == RESTARTER_STATE_OFFLINE);
 897                 break;
 898 
 899         case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
 900         case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
 901         case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
 902         case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
 903         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
 904         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
 905         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
 906         case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
 907         case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
 908                 break;
 909 
 910         default:
 911 #ifndef NDEBUG
 912                 uu_warn("%s:%d: Bad event %d.\n", __FILE__, __LINE__, e);
 913 #endif
 914                 abort();
 915         }
 916 
 917         restarter_protocol_send_event(v->gv_name, v->gv_restarter_channel, e,
 918             v->gv_reason);
 919 }
 920 
 921 static void
 922 graph_unset_restarter(graph_vertex_t *v)
 923 {
 924         assert(MUTEX_HELD(&dgraph_lock));
 925         assert(v->gv_flags & GV_CONFIGURED);
 926 
 927         vertex_send_event(v, RESTARTER_EVENT_TYPE_REMOVE_INSTANCE);
 928 
 929         if (v->gv_restarter_id != -1) {
 930                 graph_vertex_t *rv;
 931 
 932                 rv = vertex_get_by_id(v->gv_restarter_id);
 933                 graph_remove_edge(v, rv);
 934         }
 935 
 936         v->gv_restarter_id = -1;
 937         v->gv_restarter_channel = NULL;
 938 }
 939 
 940 /*
 941  * Return VERTEX_REMOVED when the vertex passed in argument is deleted from the
 942  * dgraph otherwise return VERTEX_INUSE.
 943  */
 944 static int
 945 free_if_unrefed(graph_vertex_t *v)
 946 {
 947         assert(MUTEX_HELD(&dgraph_lock));
 948 
 949         if (v->gv_refs > 0)
 950                 return (VERTEX_INUSE);
 951 
 952         if (v->gv_type == GVT_SVC &&
 953             uu_list_numnodes(v->gv_dependents) == 0 &&
 954             uu_list_numnodes(v->gv_dependencies) == 0) {
 955                 graph_remove_vertex(v);
 956                 return (VERTEX_REMOVED);
 957         } else if (v->gv_type == GVT_INST &&
 958             (v->gv_flags & GV_CONFIGURED) == 0 &&
 959             uu_list_numnodes(v->gv_dependents) == 1 &&
 960             uu_list_numnodes(v->gv_dependencies) == 0) {
 961                 remove_inst_vertex(v);
 962                 return (VERTEX_REMOVED);
 963         }
 964 
 965         return (VERTEX_INUSE);
 966 }
 967 
 968 static void
 969 delete_depgroup(graph_vertex_t *v)
 970 {
 971         graph_edge_t *e;
 972         graph_vertex_t *dv;
 973 
 974         assert(MUTEX_HELD(&dgraph_lock));
 975         assert(v->gv_type == GVT_GROUP);
 976         assert(uu_list_numnodes(v->gv_dependents) == 0);
 977 
 978         while ((e = uu_list_first(v->gv_dependencies)) != NULL) {
 979                 dv = e->ge_vertex;
 980 
 981                 graph_remove_edge(v, dv);
 982 
 983                 switch (dv->gv_type) {
 984                 case GVT_INST:          /* instance dependency */
 985                 case GVT_SVC:           /* service dependency */
 986                         (void) free_if_unrefed(dv);
 987                         break;
 988 
 989                 case GVT_FILE:          /* file dependency */
 990                         assert(uu_list_numnodes(dv->gv_dependencies) == 0);
 991                         if (uu_list_numnodes(dv->gv_dependents) == 0)
 992                                 graph_remove_vertex(dv);
 993                         break;
 994 
 995                 default:
 996 #ifndef NDEBUG
 997                         uu_warn("%s:%d: Unexpected node type %d", __FILE__,
 998                             __LINE__, dv->gv_type);
 999 #endif
1000                         abort();
1001                 }
1002         }
1003 
1004         graph_remove_vertex(v);
1005 }
1006 
1007 static int
1008 delete_instance_deps_cb(graph_edge_t *e, void **ptrs)
1009 {
1010         graph_vertex_t *v = ptrs[0];
1011         boolean_t delete_restarter_dep = (boolean_t)ptrs[1];
1012         graph_vertex_t *dv;
1013 
1014         dv = e->ge_vertex;
1015 
1016         /*
1017          * We have four possibilities here:
1018          *   - GVT_INST: restarter
1019          *   - GVT_GROUP - GVT_INST: instance dependency
1020          *   - GVT_GROUP - GVT_SVC - GV_INST: service dependency
1021          *   - GVT_GROUP - GVT_FILE: file dependency
1022          */
1023         switch (dv->gv_type) {
1024         case GVT_INST:  /* restarter */
1025                 assert(dv->gv_id == v->gv_restarter_id);
1026                 if (delete_restarter_dep)
1027                         graph_remove_edge(v, dv);
1028                 break;
1029 
1030         case GVT_GROUP: /* pg dependency */
1031                 graph_remove_edge(v, dv);
1032                 delete_depgroup(dv);
1033                 break;
1034 
1035         case GVT_FILE:
1036                 /* These are currently not direct dependencies */
1037 
1038         default:
1039 #ifndef NDEBUG
1040                 uu_warn("%s:%d: Bad vertex type %d.\n", __FILE__, __LINE__,
1041                     dv->gv_type);
1042 #endif
1043                 abort();
1044         }
1045 
1046         return (UU_WALK_NEXT);
1047 }
1048 
1049 static void
1050 delete_instance_dependencies(graph_vertex_t *v, boolean_t delete_restarter_dep)
1051 {
1052         void *ptrs[2];
1053         int r;
1054 
1055         assert(MUTEX_HELD(&dgraph_lock));
1056         assert(v->gv_type == GVT_INST);
1057 
1058         ptrs[0] = v;
1059         ptrs[1] = (void *)delete_restarter_dep;
1060 
1061         r = uu_list_walk(v->gv_dependencies,
1062             (uu_walk_fn_t *)delete_instance_deps_cb, &ptrs, UU_WALK_ROBUST);
1063         assert(r == 0);
1064 }
1065 
1066 /*
1067  * int graph_insert_vertex_unconfigured()
1068  *   Insert a vertex without sending any restarter events. If the vertex
1069  *   already exists or creation is successful, return a pointer to it in *vp.
1070  *
1071  *   If type is not GVT_GROUP, dt can remain unset.
1072  *
1073  *   Returns 0, EEXIST, or EINVAL if the arguments are invalid (i.e., fmri
1074  *   doesn't agree with type, or type doesn't agree with dt).
1075  */
1076 static int
1077 graph_insert_vertex_unconfigured(const char *fmri, gv_type_t type,
1078     depgroup_type_t dt, restarter_error_t rt, graph_vertex_t **vp)
1079 {
1080         int r;
1081         int i;
1082 
1083         assert(MUTEX_HELD(&dgraph_lock));
1084 
1085         switch (type) {
1086         case GVT_SVC:
1087         case GVT_INST:
1088                 if (strncmp(fmri, "svc:", sizeof ("svc:") - 1) != 0)
1089                         return (EINVAL);
1090                 break;
1091 
1092         case GVT_FILE:
1093                 if (strncmp(fmri, "file:", sizeof ("file:") - 1) != 0)
1094                         return (EINVAL);
1095                 break;
1096 
1097         case GVT_GROUP:
1098                 if (dt <= 0 || rt < 0)
1099                         return (EINVAL);
1100                 break;
1101 
1102         default:
1103 #ifndef NDEBUG
1104                 uu_warn("%s:%d: Unknown type %d.\n", __FILE__, __LINE__, type);
1105 #endif
1106                 abort();
1107         }
1108 
1109         *vp = vertex_get_by_name(fmri);
1110         if (*vp != NULL)
1111                 return (EEXIST);
1112 
1113         *vp = graph_add_vertex(fmri);
1114 
1115         (*vp)->gv_type = type;
1116         (*vp)->gv_depgroup = dt;
1117         (*vp)->gv_restart = rt;
1118 
1119         (*vp)->gv_flags = 0;
1120         (*vp)->gv_state = RESTARTER_STATE_NONE;
1121 
1122         for (i = 0; special_vertices[i].name != NULL; ++i) {
1123                 if (strcmp(fmri, special_vertices[i].name) == 0) {
1124                         (*vp)->gv_start_f = special_vertices[i].start_f;
1125                         (*vp)->gv_post_online_f =
1126                             special_vertices[i].post_online_f;
1127                         (*vp)->gv_post_disable_f =
1128                             special_vertices[i].post_disable_f;
1129                         break;
1130                 }
1131         }
1132 
1133         (*vp)->gv_restarter_id = -1;
1134         (*vp)->gv_restarter_channel = 0;
1135 
1136         if (type == GVT_INST) {
1137                 char *sfmri;
1138                 graph_vertex_t *sv;
1139 
1140                 sfmri = inst_fmri_to_svc_fmri(fmri);
1141                 sv = vertex_get_by_name(sfmri);
1142                 if (sv == NULL) {
1143                         r = graph_insert_vertex_unconfigured(sfmri, GVT_SVC, 0,
1144                             0, &sv);
1145                         assert(r == 0);
1146                 }
1147                 startd_free(sfmri, max_scf_fmri_size);
1148 
1149                 graph_add_edge(sv, *vp);
1150         }
1151 
1152         /*
1153          * If this vertex is in the subgraph, mark it as so, for both
1154          * GVT_INST and GVT_SERVICE verteces.
1155          * A GVT_SERVICE vertex can only be in the subgraph if another instance
1156          * depends on it, in which case it's already been added to the graph
1157          * and marked as in the subgraph (by refresh_vertex()).  If a
1158          * GVT_SERVICE vertex was freshly added (by the code above), it means
1159          * that it has no dependents, and cannot be in the subgraph.
1160          * Regardless of this, we still check that gv_flags includes
1161          * GV_INSUBGRAPH in the event that future behavior causes the above
1162          * code to add a GVT_SERVICE vertex which should be in the subgraph.
1163          */
1164 
1165         (*vp)->gv_flags |= (should_be_in_subgraph(*vp)? GV_INSUBGRAPH : 0);
1166 
1167         return (0);
1168 }
1169 
1170 /*
1171  * Returns 0 on success or ELOOP if the dependency would create a cycle.
1172  */
1173 static int
1174 graph_insert_dependency(graph_vertex_t *fv, graph_vertex_t *tv, int **pathp)
1175 {
1176         hrtime_t now;
1177 
1178         assert(MUTEX_HELD(&dgraph_lock));
1179 
1180         /* cycle detection */
1181         now = gethrtime();
1182 
1183         /* Don't follow exclusions. */
1184         if (!(fv->gv_type == GVT_GROUP &&
1185             fv->gv_depgroup == DEPGRP_EXCLUDE_ALL)) {
1186                 *pathp = is_path_to(tv, fv);
1187                 if (*pathp)
1188                         return (ELOOP);
1189         }
1190 
1191         dep_cycle_ns += gethrtime() - now;
1192         ++dep_inserts;
1193         now = gethrtime();
1194 
1195         graph_add_edge(fv, tv);
1196 
1197         dep_insert_ns += gethrtime() - now;
1198 
1199         /* Check if the dependency adds the "to" vertex to the subgraph */
1200         tv->gv_flags |= (should_be_in_subgraph(tv) ? GV_INSUBGRAPH : 0);
1201 
1202         return (0);
1203 }
1204 
1205 static int
1206 inst_running(graph_vertex_t *v)
1207 {
1208         assert(v->gv_type == GVT_INST);
1209 
1210         if (v->gv_state == RESTARTER_STATE_ONLINE ||
1211             v->gv_state == RESTARTER_STATE_DEGRADED)
1212                 return (1);
1213 
1214         return (0);
1215 }
1216 
1217 /*
1218  * The dependency evaluation functions return
1219  *   1 - dependency satisfied
1220  *   0 - dependency unsatisfied
1221  *   -1 - dependency unsatisfiable (without administrator intervention)
1222  *
1223  * The functions also take a boolean satbility argument.  When true, the
1224  * functions may recurse in order to determine satisfiability.
1225  */
1226 static int require_any_satisfied(graph_vertex_t *, boolean_t);
1227 static int dependency_satisfied(graph_vertex_t *, boolean_t);
1228 
1229 /*
1230  * A require_all dependency is unsatisfied if any elements are unsatisfied.  It
1231  * is unsatisfiable if any elements are unsatisfiable.
1232  */
1233 static int
1234 require_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1235 {
1236         graph_edge_t *edge;
1237         int i;
1238         boolean_t any_unsatisfied;
1239 
1240         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1241                 return (1);
1242 
1243         any_unsatisfied = B_FALSE;
1244 
1245         for (edge = uu_list_first(groupv->gv_dependencies);
1246             edge != NULL;
1247             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1248                 i = dependency_satisfied(edge->ge_vertex, satbility);
1249                 if (i == 1)
1250                         continue;
1251 
1252                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1253                     "require_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1254                     edge->ge_vertex->gv_name, i == 0 ? "ed" : "able");
1255 
1256                 if (!satbility)
1257                         return (0);
1258 
1259                 if (i == -1)
1260                         return (-1);
1261 
1262                 any_unsatisfied = B_TRUE;
1263         }
1264 
1265         return (any_unsatisfied ? 0 : 1);
1266 }
1267 
1268 /*
1269  * A require_any dependency is satisfied if any element is satisfied.  It is
1270  * satisfiable if any element is satisfiable.
1271  */
1272 static int
1273 require_any_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1274 {
1275         graph_edge_t *edge;
1276         int s;
1277         boolean_t satisfiable;
1278 
1279         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1280                 return (1);
1281 
1282         satisfiable = B_FALSE;
1283 
1284         for (edge = uu_list_first(groupv->gv_dependencies);
1285             edge != NULL;
1286             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1287                 s = dependency_satisfied(edge->ge_vertex, satbility);
1288 
1289                 if (s == 1)
1290                         return (1);
1291 
1292                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1293                     "require_any(%s): %s is unsatisfi%s.\n",
1294                     groupv->gv_name, edge->ge_vertex->gv_name,
1295                     s == 0 ? "ed" : "able");
1296 
1297                 if (satbility && s == 0)
1298                         satisfiable = B_TRUE;
1299         }
1300 
1301         if (!satbility)
1302                 return (0);
1303 
1304         return (satisfiable ? 0 : -1);
1305 }
1306 
1307 /*
1308  * An optional_all dependency only considers elements which are configured,
1309  * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1310  * is unsatisfied.
1311  *
1312  * Offline dependencies which are waiting for a dependency to come online are
1313  * unsatisfied.  Offline dependences which cannot possibly come online
1314  * (unsatisfiable) are always considered satisfied.
1315  */
1316 static int
1317 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1318 {
1319         graph_edge_t *edge;
1320         graph_vertex_t *v;
1321         boolean_t any_qualified;
1322         boolean_t any_unsatisfied;
1323         int i;
1324 
1325         any_qualified = B_FALSE;
1326         any_unsatisfied = B_FALSE;
1327 
1328         for (edge = uu_list_first(groupv->gv_dependencies);
1329             edge != NULL;
1330             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1331                 v = edge->ge_vertex;
1332 
1333                 switch (v->gv_type) {
1334                 case GVT_INST:
1335                         /* Skip missing or disabled instances */
1336                         if ((v->gv_flags & (GV_CONFIGURED | GV_ENABLED)) !=
1337                             (GV_CONFIGURED | GV_ENABLED))
1338                                 continue;
1339 
1340                         if (v->gv_state == RESTARTER_STATE_MAINT)
1341                                 continue;
1342 
1343                         if (v->gv_flags & GV_TOOFFLINE)
1344                                 continue;
1345 
1346                         any_qualified = B_TRUE;
1347                         if (v->gv_state == RESTARTER_STATE_OFFLINE) {
1348                                 /*
1349                                  * For offline dependencies, treat unsatisfiable
1350                                  * as satisfied.
1351                                  */
1352                                 i = dependency_satisfied(v, B_TRUE);
1353                                 if (i == -1)
1354                                         i = 1;
1355                         } else if (v->gv_state == RESTARTER_STATE_DISABLED) {
1356                                 /*
1357                                  * The service is enabled, but hasn't
1358                                  * transitioned out of disabled yet.  Treat it
1359                                  * as unsatisfied (not unsatisfiable).
1360                                  */
1361                                 i = 0;
1362                         } else {
1363                                 i = dependency_satisfied(v, satbility);
1364                         }
1365                         break;
1366 
1367                 case GVT_FILE:
1368                         any_qualified = B_TRUE;
1369                         i = dependency_satisfied(v, satbility);
1370 
1371                         break;
1372 
1373                 case GVT_SVC: {
1374                         boolean_t svc_any_qualified;
1375                         boolean_t svc_satisfied;
1376                         boolean_t svc_satisfiable;
1377                         graph_vertex_t *v2;
1378                         graph_edge_t *e2;
1379 
1380                         svc_any_qualified = B_FALSE;
1381                         svc_satisfied = B_FALSE;
1382                         svc_satisfiable = B_FALSE;
1383 
1384                         for (e2 = uu_list_first(v->gv_dependencies);
1385                             e2 != NULL;
1386                             e2 = uu_list_next(v->gv_dependencies, e2)) {
1387                                 v2 = e2->ge_vertex;
1388                                 assert(v2->gv_type == GVT_INST);
1389 
1390                                 if ((v2->gv_flags &
1391                                     (GV_CONFIGURED | GV_ENABLED)) !=
1392                                     (GV_CONFIGURED | GV_ENABLED))
1393                                         continue;
1394 
1395                                 if (v2->gv_state == RESTARTER_STATE_MAINT)
1396                                         continue;
1397 
1398                                 if (v2->gv_flags & GV_TOOFFLINE)
1399                                         continue;
1400 
1401                                 svc_any_qualified = B_TRUE;
1402 
1403                                 if (v2->gv_state == RESTARTER_STATE_OFFLINE) {
1404                                         /*
1405                                          * For offline dependencies, treat
1406                                          * unsatisfiable as satisfied.
1407                                          */
1408                                         i = dependency_satisfied(v2, B_TRUE);
1409                                         if (i == -1)
1410                                                 i = 1;
1411                                 } else if (v2->gv_state ==
1412                                     RESTARTER_STATE_DISABLED) {
1413                                         i = 0;
1414                                 } else {
1415                                         i = dependency_satisfied(v2, satbility);
1416                                 }
1417 
1418                                 if (i == 1) {
1419                                         svc_satisfied = B_TRUE;
1420                                         break;
1421                                 }
1422                                 if (i == 0)
1423                                         svc_satisfiable = B_TRUE;
1424                         }
1425 
1426                         if (!svc_any_qualified)
1427                                 continue;
1428                         any_qualified = B_TRUE;
1429                         if (svc_satisfied) {
1430                                 i = 1;
1431                         } else if (svc_satisfiable) {
1432                                 i = 0;
1433                         } else {
1434                                 i = -1;
1435                         }
1436                         break;
1437                 }
1438 
1439                 case GVT_GROUP:
1440                 default:
1441 #ifndef NDEBUG
1442                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1443                             __LINE__, v->gv_type);
1444 #endif
1445                         abort();
1446                 }
1447 
1448                 if (i == 1)
1449                         continue;
1450 
1451                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1452                     "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1453                     v->gv_name, i == 0 ? "ed" : "able");
1454 
1455                 if (!satbility)
1456                         return (0);
1457                 if (i == -1)
1458                         return (-1);
1459                 any_unsatisfied = B_TRUE;
1460         }
1461 
1462         if (!any_qualified)
1463                 return (1);
1464 
1465         return (any_unsatisfied ? 0 : 1);
1466 }
1467 
1468 /*
1469  * An exclude_all dependency is unsatisfied if any non-service element is
1470  * satisfied or any service instance which is configured, enabled, and not in
1471  * maintenance is satisfied.  Usually when unsatisfied, it is also
1472  * unsatisfiable.
1473  */
1474 #define LOG_EXCLUDE(u, v)                                               \
1475         log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,                   \
1476             "exclude_all(%s): %s is satisfied.\n",                      \
1477             (u)->gv_name, (v)->gv_name)
1478 
1479 /* ARGSUSED */
1480 static int
1481 exclude_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1482 {
1483         graph_edge_t *edge, *e2;
1484         graph_vertex_t *v, *v2;
1485 
1486         for (edge = uu_list_first(groupv->gv_dependencies);
1487             edge != NULL;
1488             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1489                 v = edge->ge_vertex;
1490 
1491                 switch (v->gv_type) {
1492                 case GVT_INST:
1493                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1494                                 continue;
1495 
1496                         switch (v->gv_state) {
1497                         case RESTARTER_STATE_ONLINE:
1498                         case RESTARTER_STATE_DEGRADED:
1499                                 LOG_EXCLUDE(groupv, v);
1500                                 return (v->gv_flags & GV_ENABLED ? -1 : 0);
1501 
1502                         case RESTARTER_STATE_OFFLINE:
1503                         case RESTARTER_STATE_UNINIT:
1504                                 LOG_EXCLUDE(groupv, v);
1505                                 return (0);
1506 
1507                         case RESTARTER_STATE_DISABLED:
1508                         case RESTARTER_STATE_MAINT:
1509                                 continue;
1510 
1511                         default:
1512 #ifndef NDEBUG
1513                                 uu_warn("%s:%d: Unexpected vertex state %d.\n",
1514                                     __FILE__, __LINE__, v->gv_state);
1515 #endif
1516                                 abort();
1517                         }
1518                         /* NOTREACHED */
1519 
1520                 case GVT_SVC:
1521                         break;
1522 
1523                 case GVT_FILE:
1524                         if (!file_ready(v))
1525                                 continue;
1526                         LOG_EXCLUDE(groupv, v);
1527                         return (-1);
1528 
1529                 case GVT_GROUP:
1530                 default:
1531 #ifndef NDEBUG
1532                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1533                             __LINE__, v->gv_type);
1534 #endif
1535                         abort();
1536                 }
1537 
1538                 /* v represents a service */
1539                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1540                         continue;
1541 
1542                 for (e2 = uu_list_first(v->gv_dependencies);
1543                     e2 != NULL;
1544                     e2 = uu_list_next(v->gv_dependencies, e2)) {
1545                         v2 = e2->ge_vertex;
1546                         assert(v2->gv_type == GVT_INST);
1547 
1548                         if ((v2->gv_flags & GV_CONFIGURED) == 0)
1549                                 continue;
1550 
1551                         switch (v2->gv_state) {
1552                         case RESTARTER_STATE_ONLINE:
1553                         case RESTARTER_STATE_DEGRADED:
1554                                 LOG_EXCLUDE(groupv, v2);
1555                                 return (v2->gv_flags & GV_ENABLED ? -1 : 0);
1556 
1557                         case RESTARTER_STATE_OFFLINE:
1558                         case RESTARTER_STATE_UNINIT:
1559                                 LOG_EXCLUDE(groupv, v2);
1560                                 return (0);
1561 
1562                         case RESTARTER_STATE_DISABLED:
1563                         case RESTARTER_STATE_MAINT:
1564                                 continue;
1565 
1566                         default:
1567 #ifndef NDEBUG
1568                                 uu_warn("%s:%d: Unexpected vertex type %d.\n",
1569                                     __FILE__, __LINE__, v2->gv_type);
1570 #endif
1571                                 abort();
1572                         }
1573                 }
1574         }
1575 
1576         return (1);
1577 }
1578 
1579 /*
1580  * int instance_satisfied()
1581  *   Determine if all the dependencies are satisfied for the supplied instance
1582  *   vertex. Return 1 if they are, 0 if they aren't, and -1 if they won't be
1583  *   without administrator intervention.
1584  */
1585 static int
1586 instance_satisfied(graph_vertex_t *v, boolean_t satbility)
1587 {
1588         assert(v->gv_type == GVT_INST);
1589         assert(!inst_running(v));
1590 
1591         return (require_all_satisfied(v, satbility));
1592 }
1593 
1594 /*
1595  * Decide whether v can satisfy a dependency.  v can either be a child of
1596  * a group vertex, or of an instance vertex.
1597  */
1598 static int
1599 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1600 {
1601         switch (v->gv_type) {
1602         case GVT_INST:
1603                 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1604                         if (v->gv_flags & GV_DEATHROW) {
1605                                 /*
1606                                  * A dependency on an instance with GV_DEATHROW
1607                                  * flag is always considered as satisfied.
1608                                  */
1609                                 return (1);
1610                         }
1611                         return (-1);
1612                 }
1613 
1614                 /*
1615                  * Any vertex with the GV_TOOFFLINE flag set is guaranteed
1616                  * to have its dependencies unsatisfiable.
1617                  */
1618                 if (v->gv_flags & GV_TOOFFLINE)
1619                         return (-1);
1620 
1621                 switch (v->gv_state) {
1622                 case RESTARTER_STATE_ONLINE:
1623                 case RESTARTER_STATE_DEGRADED:
1624                         return (1);
1625 
1626                 case RESTARTER_STATE_OFFLINE:
1627                         if (!satbility)
1628                                 return (0);
1629                         return (instance_satisfied(v, satbility) != -1 ?
1630                             0 : -1);
1631 
1632                 case RESTARTER_STATE_DISABLED:
1633                 case RESTARTER_STATE_MAINT:
1634                         return (-1);
1635 
1636                 case RESTARTER_STATE_UNINIT:
1637                         return (0);
1638 
1639                 default:
1640 #ifndef NDEBUG
1641                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
1642                             __FILE__, __LINE__, v->gv_state);
1643 #endif
1644                         abort();
1645                         /* NOTREACHED */
1646                 }
1647 
1648         case GVT_SVC:
1649                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1650                         return (-1);
1651                 return (require_any_satisfied(v, satbility));
1652 
1653         case GVT_FILE:
1654                 /* i.e., we assume files will not be automatically generated */
1655                 return (file_ready(v) ? 1 : -1);
1656 
1657         case GVT_GROUP:
1658                 break;
1659 
1660         default:
1661 #ifndef NDEBUG
1662                 uu_warn("%s:%d: Unexpected node type %d.\n", __FILE__, __LINE__,
1663                     v->gv_type);
1664 #endif
1665                 abort();
1666                 /* NOTREACHED */
1667         }
1668 
1669         switch (v->gv_depgroup) {
1670         case DEPGRP_REQUIRE_ANY:
1671                 return (require_any_satisfied(v, satbility));
1672 
1673         case DEPGRP_REQUIRE_ALL:
1674                 return (require_all_satisfied(v, satbility));
1675 
1676         case DEPGRP_OPTIONAL_ALL:
1677                 return (optional_all_satisfied(v, satbility));
1678 
1679         case DEPGRP_EXCLUDE_ALL:
1680                 return (exclude_all_satisfied(v, satbility));
1681 
1682         default:
1683 #ifndef NDEBUG
1684                 uu_warn("%s:%d: Unknown dependency grouping %d.\n", __FILE__,
1685                     __LINE__, v->gv_depgroup);
1686 #endif
1687                 abort();
1688         }
1689 }
1690 
1691 void
1692 graph_start_if_satisfied(graph_vertex_t *v)
1693 {
1694         if (v->gv_state == RESTARTER_STATE_OFFLINE &&
1695             instance_satisfied(v, B_FALSE) == 1) {
1696                 if (v->gv_start_f == NULL)
1697                         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
1698                 else
1699                         v->gv_start_f(v);
1700         }
1701 }
1702 
1703 /*
1704  * propagate_satbility()
1705  *
1706  * This function is used when the given vertex changes state in such a way that
1707  * one of its dependents may become unsatisfiable.  This happens when an
1708  * instance transitions between offline -> online, or from !running ->
1709  * maintenance, as well as when an instance is removed from the graph.
1710  *
1711  * We have to walk all the dependents, since optional_all dependencies several
1712  * levels up could become (un)satisfied, instead of unsatisfiable.  For example,
1713  *
1714  *      +-----+  optional_all  +-----+  require_all  +-----+
1715  *      |  A  |--------------->|  B  |-------------->|  C  |
1716  *      +-----+                +-----+               +-----+
1717  *
1718  *                                              offline -> maintenance
1719  *
1720  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1721  * an optional dependency, what was previously an unsatisfiable situation is now
1722  * satisfied (B will never come online, even though its state hasn't changed).
1723  *
1724  * Note that it's not necessary to continue examining dependents after reaching
1725  * an optional_all dependency.  It's not possible for an optional_all dependency
1726  * to change satisfiability without also coming online, in which case we get a
1727  * start event and propagation continues naturally.  However, it does no harm to
1728  * continue propagating satisfiability (as it is a relatively rare event), and
1729  * keeps the walker code simple and generic.
1730  */
1731 /*ARGSUSED*/
1732 static int
1733 satbility_cb(graph_vertex_t *v, void *arg)
1734 {
1735         if (v->gv_type == GVT_INST)
1736                 graph_start_if_satisfied(v);
1737 
1738         return (UU_WALK_NEXT);
1739 }
1740 
1741 static void
1742 propagate_satbility(graph_vertex_t *v)
1743 {
1744         graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1745 }
1746 
1747 static void propagate_stop(graph_vertex_t *, void *);
1748 
1749 /* ARGSUSED */
1750 static void
1751 propagate_start(graph_vertex_t *v, void *arg)
1752 {
1753         switch (v->gv_type) {
1754         case GVT_INST:
1755                 graph_start_if_satisfied(v);
1756                 break;
1757 
1758         case GVT_GROUP:
1759                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1760                         graph_walk_dependents(v, propagate_stop,
1761                             (void *)RERR_RESTART);
1762                         break;
1763                 }
1764                 /* FALLTHROUGH */
1765 
1766         case GVT_SVC:
1767                 graph_walk_dependents(v, propagate_start, NULL);
1768                 break;
1769 
1770         case GVT_FILE:
1771 #ifndef NDEBUG
1772                 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1773                     __FILE__, __LINE__);
1774 #endif
1775                 abort();
1776                 /* NOTREACHED */
1777 
1778         default:
1779 #ifndef NDEBUG
1780                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1781                     v->gv_type);
1782 #endif
1783                 abort();
1784         }
1785 }
1786 
1787 static void
1788 propagate_stop(graph_vertex_t *v, void *arg)
1789 {
1790         graph_edge_t *e;
1791         graph_vertex_t *svc;
1792         restarter_error_t err = (restarter_error_t)arg;
1793 
1794         switch (v->gv_type) {
1795         case GVT_INST:
1796                 /* Restarter */
1797                 if (err > RERR_NONE && inst_running(v)) {
1798                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1799                                 vertex_send_event(v,
1800                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1801                         } else {
1802                                 vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1803                         }
1804                 }
1805                 break;
1806 
1807         case GVT_SVC:
1808                 graph_walk_dependents(v, propagate_stop, arg);
1809                 break;
1810 
1811         case GVT_FILE:
1812 #ifndef NDEBUG
1813                 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1814                     __FILE__, __LINE__);
1815 #endif
1816                 abort();
1817                 /* NOTREACHED */
1818 
1819         case GVT_GROUP:
1820                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1821                         graph_walk_dependents(v, propagate_start, NULL);
1822                         break;
1823                 }
1824 
1825                 if (err == RERR_NONE || err > v->gv_restart)
1826                         break;
1827 
1828                 assert(uu_list_numnodes(v->gv_dependents) == 1);
1829                 e = uu_list_first(v->gv_dependents);
1830                 svc = e->ge_vertex;
1831 
1832                 if (inst_running(svc)) {
1833                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1834                                 vertex_send_event(svc,
1835                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1836                         } else {
1837                                 vertex_send_event(svc,
1838                                     RESTARTER_EVENT_TYPE_STOP);
1839                         }
1840                 }
1841                 break;
1842 
1843         default:
1844 #ifndef NDEBUG
1845                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1846                     v->gv_type);
1847 #endif
1848                 abort();
1849         }
1850 }
1851 
1852 void
1853 offline_vertex(graph_vertex_t *v)
1854 {
1855         scf_handle_t *h = libscf_handle_create_bound_loop();
1856         scf_instance_t *scf_inst = safe_scf_instance_create(h);
1857         scf_propertygroup_t *pg = safe_scf_pg_create(h);
1858         restarter_instance_state_t state, next_state;
1859         int r;
1860 
1861         assert(v->gv_type == GVT_INST);
1862 
1863         if (scf_inst == NULL)
1864                 bad_error("safe_scf_instance_create", scf_error());
1865         if (pg == NULL)
1866                 bad_error("safe_scf_pg_create", scf_error());
1867 
1868         /* if the vertex is already going offline, return */
1869 rep_retry:
1870         if (scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, scf_inst, NULL,
1871             NULL, SCF_DECODE_FMRI_EXACT) != 0) {
1872                 switch (scf_error()) {
1873                 case SCF_ERROR_CONNECTION_BROKEN:
1874                         libscf_handle_rebind(h);
1875                         goto rep_retry;
1876 
1877                 case SCF_ERROR_NOT_FOUND:
1878                         scf_pg_destroy(pg);
1879                         scf_instance_destroy(scf_inst);
1880                         (void) scf_handle_unbind(h);
1881                         scf_handle_destroy(h);
1882                         return;
1883                 }
1884                 uu_die("Can't decode FMRI %s: %s\n", v->gv_name,
1885                     scf_strerror(scf_error()));
1886         }
1887 
1888         r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
1889         if (r != 0) {
1890                 switch (scf_error()) {
1891                 case SCF_ERROR_CONNECTION_BROKEN:
1892                         libscf_handle_rebind(h);
1893                         goto rep_retry;
1894 
1895                 case SCF_ERROR_NOT_SET:
1896                 case SCF_ERROR_NOT_FOUND:
1897                         scf_pg_destroy(pg);
1898                         scf_instance_destroy(scf_inst);
1899                         (void) scf_handle_unbind(h);
1900                         scf_handle_destroy(h);
1901                         return;
1902 
1903                 default:
1904                         bad_error("scf_instance_get_pg", scf_error());
1905                 }
1906         } else {
1907                 r = libscf_read_states(pg, &state, &next_state);
1908                 if (r == 0 && (next_state == RESTARTER_STATE_OFFLINE ||
1909                     next_state == RESTARTER_STATE_DISABLED)) {
1910                         log_framework(LOG_DEBUG,
1911                             "%s: instance is already going down.\n",
1912                             v->gv_name);
1913                         scf_pg_destroy(pg);
1914                         scf_instance_destroy(scf_inst);
1915                         (void) scf_handle_unbind(h);
1916                         scf_handle_destroy(h);
1917                         return;
1918                 }
1919         }
1920 
1921         scf_pg_destroy(pg);
1922         scf_instance_destroy(scf_inst);
1923         (void) scf_handle_unbind(h);
1924         scf_handle_destroy(h);
1925 
1926         vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP_RESET);
1927 }
1928 
1929 /*
1930  * void graph_enable_by_vertex()
1931  *   If admin is non-zero, this is an administrative request for change
1932  *   of the enabled property.  Thus, send the ADMIN_DISABLE rather than
1933  *   a plain DISABLE restarter event.
1934  */
1935 void
1936 graph_enable_by_vertex(graph_vertex_t *vertex, int enable, int admin)
1937 {
1938         graph_vertex_t *v;
1939         int r;
1940 
1941         assert(MUTEX_HELD(&dgraph_lock));
1942         assert((vertex->gv_flags & GV_CONFIGURED));
1943 
1944         vertex->gv_flags = (vertex->gv_flags & ~GV_ENABLED) |
1945             (enable ? GV_ENABLED : 0);
1946 
1947         if (enable) {
1948                 if (vertex->gv_state != RESTARTER_STATE_OFFLINE &&
1949                     vertex->gv_state != RESTARTER_STATE_DEGRADED &&
1950                     vertex->gv_state != RESTARTER_STATE_ONLINE) {
1951                         /*
1952                          * In case the vertex was notified to go down,
1953                          * but now can return online, clear the _TOOFFLINE
1954                          * and _TODISABLE flags.
1955                          */
1956                         vertex->gv_flags &= ~GV_TOOFFLINE;
1957                         vertex->gv_flags &= ~GV_TODISABLE;
1958 
1959                         vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ENABLE);
1960                 }
1961 
1962                 /*
1963                  * Wait for state update from restarter before sending _START or
1964                  * _STOP.
1965                  */
1966 
1967                 return;
1968         }
1969 
1970         if (vertex->gv_state == RESTARTER_STATE_DISABLED)
1971                 return;
1972 
1973         if (!admin) {
1974                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_DISABLE);
1975 
1976                 /*
1977                  * Wait for state update from restarter before sending _START or
1978                  * _STOP.
1979                  */
1980 
1981                 return;
1982         }
1983 
1984         /*
1985          * If it is a DISABLE event requested by the administrator then we are
1986          * offlining the dependents first.
1987          */
1988 
1989         /*
1990          * Set GV_TOOFFLINE for the services we are offlining. We cannot
1991          * clear the GV_TOOFFLINE bits from all the services because
1992          * other DISABLE events might be handled at the same time.
1993          */
1994         vertex->gv_flags |= GV_TOOFFLINE;
1995 
1996         /* remember which vertex to disable... */
1997         vertex->gv_flags |= GV_TODISABLE;
1998 
1999         log_framework(LOG_DEBUG, "Marking in-subtree vertices before "
2000             "disabling %s.\n", vertex->gv_name);
2001 
2002         /* set GV_TOOFFLINE for its dependents */
2003         r = uu_list_walk(vertex->gv_dependents, (uu_walk_fn_t *)mark_subtree,
2004             NULL, 0);
2005         assert(r == 0);
2006 
2007         /* disable the instance now if there is nothing else to offline */
2008         if (insubtree_dependents_down(vertex) == B_TRUE) {
2009                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
2010                 return;
2011         }
2012 
2013         /*
2014          * This loop is similar to the one used for the graph reversal shutdown
2015          * and could be improved in term of performance for the subtree reversal
2016          * disable case.
2017          */
2018         for (v = uu_list_first(dgraph); v != NULL;
2019             v = uu_list_next(dgraph, v)) {
2020                 /* skip the vertex we are disabling for now */
2021                 if (v == vertex)
2022                         continue;
2023 
2024                 if (v->gv_type != GVT_INST ||
2025                     (v->gv_flags & GV_CONFIGURED) == 0 ||
2026                     (v->gv_flags & GV_ENABLED) == 0 ||
2027                     (v->gv_flags & GV_TOOFFLINE) == 0)
2028                         continue;
2029 
2030                 if ((v->gv_state != RESTARTER_STATE_ONLINE) &&
2031                     (v->gv_state != RESTARTER_STATE_DEGRADED)) {
2032                         /* continue if there is nothing to offline */
2033                         continue;
2034                 }
2035 
2036                 /*
2037                  * Instances which are up need to come down before we're
2038                  * done, but we can only offline the leaves here. An
2039                  * instance is a leaf when all its dependents are down.
2040                  */
2041                 if (insubtree_dependents_down(v) == B_TRUE) {
2042                         log_framework(LOG_DEBUG, "Offlining in-subtree "
2043                             "instance %s for %s.\n",
2044                             v->gv_name, vertex->gv_name);
2045                         offline_vertex(v);
2046                 }
2047         }
2048 }
2049 
2050 static int configure_vertex(graph_vertex_t *, scf_instance_t *);
2051 
2052 /*
2053  * Set the restarter for v to fmri_arg.  That is, make sure a vertex for
2054  * fmri_arg exists, make v depend on it, and send _ADD_INSTANCE for v.  If
2055  * v is already configured and fmri_arg indicates the current restarter, do
2056  * nothing.  If v is configured and fmri_arg is a new restarter, delete v's
2057  * dependency on the restarter, send _REMOVE_INSTANCE for v, and set the new
2058  * restarter.  Returns 0 on success, EINVAL if the FMRI is invalid,
2059  * ECONNABORTED if the repository connection is broken, and ELOOP
2060  * if the dependency would create a cycle.  In the last case, *pathp will
2061  * point to a -1-terminated array of ids which compose the path from v to
2062  * restarter_fmri.
2063  */
2064 int
2065 graph_change_restarter(graph_vertex_t *v, const char *fmri_arg, scf_handle_t *h,
2066     int **pathp)
2067 {
2068         char *restarter_fmri = NULL;
2069         graph_vertex_t *rv;
2070         int err;
2071         int id;
2072 
2073         assert(MUTEX_HELD(&dgraph_lock));
2074 
2075         if (fmri_arg[0] != '\0') {
2076                 err = fmri_canonify(fmri_arg, &restarter_fmri, B_TRUE);
2077                 if (err != 0) {
2078                         assert(err == EINVAL);
2079                         return (err);
2080                 }
2081         }
2082 
2083         if (restarter_fmri == NULL ||
2084             strcmp(restarter_fmri, SCF_SERVICE_STARTD) == 0) {
2085                 if (v->gv_flags & GV_CONFIGURED) {
2086                         if (v->gv_restarter_id == -1) {
2087                                 if (restarter_fmri != NULL)
2088                                         startd_free(restarter_fmri,
2089                                             max_scf_fmri_size);
2090                                 return (0);
2091                         }
2092 
2093                         graph_unset_restarter(v);
2094                 }
2095 
2096                 /* Master restarter, nothing to do. */
2097                 v->gv_restarter_id = -1;
2098                 v->gv_restarter_channel = NULL;
2099                 vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2100                 return (0);
2101         }
2102 
2103         if (v->gv_flags & GV_CONFIGURED) {
2104                 id = dict_lookup_byname(restarter_fmri);
2105                 if (id != -1 && v->gv_restarter_id == id) {
2106                         startd_free(restarter_fmri, max_scf_fmri_size);
2107                         return (0);
2108                 }
2109 
2110                 graph_unset_restarter(v);
2111         }
2112 
2113         err = graph_insert_vertex_unconfigured(restarter_fmri, GVT_INST, 0,
2114             RERR_NONE, &rv);
2115         startd_free(restarter_fmri, max_scf_fmri_size);
2116         assert(err == 0 || err == EEXIST);
2117 
2118         if (rv->gv_delegate_initialized == 0) {
2119                 if ((rv->gv_delegate_channel = restarter_protocol_init_delegate(
2120                     rv->gv_name)) == NULL)
2121                         return (EINVAL);
2122                 rv->gv_delegate_initialized = 1;
2123         }
2124         v->gv_restarter_id = rv->gv_id;
2125         v->gv_restarter_channel = rv->gv_delegate_channel;
2126 
2127         err = graph_insert_dependency(v, rv, pathp);
2128         if (err != 0) {
2129                 assert(err == ELOOP);
2130                 return (ELOOP);
2131         }
2132 
2133         vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2134 
2135         if (!(rv->gv_flags & GV_CONFIGURED)) {
2136                 scf_instance_t *inst;
2137 
2138                 err = libscf_fmri_get_instance(h, rv->gv_name, &inst);
2139                 switch (err) {
2140                 case 0:
2141                         err = configure_vertex(rv, inst);
2142                         scf_instance_destroy(inst);
2143                         switch (err) {
2144                         case 0:
2145                         case ECANCELED:
2146                                 break;
2147 
2148                         case ECONNABORTED:
2149                                 return (ECONNABORTED);
2150 
2151                         default:
2152                                 bad_error("configure_vertex", err);
2153                         }
2154                         break;
2155 
2156                 case ECONNABORTED:
2157                         return (ECONNABORTED);
2158 
2159                 case ENOENT:
2160                         break;
2161 
2162                 case ENOTSUP:
2163                         /*
2164                          * The fmri doesn't specify an instance - translate
2165                          * to EINVAL.
2166                          */
2167                         return (EINVAL);
2168 
2169                 case EINVAL:
2170                 default:
2171                         bad_error("libscf_fmri_get_instance", err);
2172                 }
2173         }
2174 
2175         return (0);
2176 }
2177 
2178 
2179 /*
2180  * Add all of the instances of the service named by fmri to the graph.
2181  * Returns
2182  *   0 - success
2183  *   ENOENT - service indicated by fmri does not exist
2184  *
2185  * In both cases *reboundp will be B_TRUE if the handle was rebound, or B_FALSE
2186  * otherwise.
2187  */
2188 static int
2189 add_service(const char *fmri, scf_handle_t *h, boolean_t *reboundp)
2190 {
2191         scf_service_t *svc;
2192         scf_instance_t *inst;
2193         scf_iter_t *iter;
2194         char *inst_fmri;
2195         int ret, r;
2196 
2197         *reboundp = B_FALSE;
2198 
2199         svc = safe_scf_service_create(h);
2200         inst = safe_scf_instance_create(h);
2201         iter = safe_scf_iter_create(h);
2202         inst_fmri = startd_alloc(max_scf_fmri_size);
2203 
2204 rebound:
2205         if (scf_handle_decode_fmri(h, fmri, NULL, svc, NULL, NULL, NULL,
2206             SCF_DECODE_FMRI_EXACT) != 0) {
2207                 switch (scf_error()) {
2208                 case SCF_ERROR_CONNECTION_BROKEN:
2209                 default:
2210                         libscf_handle_rebind(h);
2211                         *reboundp = B_TRUE;
2212                         goto rebound;
2213 
2214                 case SCF_ERROR_NOT_FOUND:
2215                         ret = ENOENT;
2216                         goto out;
2217 
2218                 case SCF_ERROR_INVALID_ARGUMENT:
2219                 case SCF_ERROR_CONSTRAINT_VIOLATED:
2220                 case SCF_ERROR_NOT_BOUND:
2221                 case SCF_ERROR_HANDLE_MISMATCH:
2222                         bad_error("scf_handle_decode_fmri", scf_error());
2223                 }
2224         }
2225 
2226         if (scf_iter_service_instances(iter, svc) != 0) {
2227                 switch (scf_error()) {
2228                 case SCF_ERROR_CONNECTION_BROKEN:
2229                 default:
2230                         libscf_handle_rebind(h);
2231                         *reboundp = B_TRUE;
2232                         goto rebound;
2233 
2234                 case SCF_ERROR_DELETED:
2235                         ret = ENOENT;
2236                         goto out;
2237 
2238                 case SCF_ERROR_HANDLE_MISMATCH:
2239                 case SCF_ERROR_NOT_BOUND:
2240                 case SCF_ERROR_NOT_SET:
2241                         bad_error("scf_iter_service_instances", scf_error());
2242                 }
2243         }
2244 
2245         for (;;) {
2246                 r = scf_iter_next_instance(iter, inst);
2247                 if (r == 0)
2248                         break;
2249                 if (r != 1) {
2250                         switch (scf_error()) {
2251                         case SCF_ERROR_CONNECTION_BROKEN:
2252                         default:
2253                                 libscf_handle_rebind(h);
2254                                 *reboundp = B_TRUE;
2255                                 goto rebound;
2256 
2257                         case SCF_ERROR_DELETED:
2258                                 ret = ENOENT;
2259                                 goto out;
2260 
2261                         case SCF_ERROR_HANDLE_MISMATCH:
2262                         case SCF_ERROR_NOT_BOUND:
2263                         case SCF_ERROR_NOT_SET:
2264                         case SCF_ERROR_INVALID_ARGUMENT:
2265                                 bad_error("scf_iter_next_instance",
2266                                     scf_error());
2267                         }
2268                 }
2269 
2270                 if (scf_instance_to_fmri(inst, inst_fmri, max_scf_fmri_size) <
2271                     0) {
2272                         switch (scf_error()) {
2273                         case SCF_ERROR_CONNECTION_BROKEN:
2274                                 libscf_handle_rebind(h);
2275                                 *reboundp = B_TRUE;
2276                                 goto rebound;
2277 
2278                         case SCF_ERROR_DELETED:
2279                                 continue;
2280 
2281                         case SCF_ERROR_NOT_BOUND:
2282                         case SCF_ERROR_NOT_SET:
2283                                 bad_error("scf_instance_to_fmri", scf_error());
2284                         }
2285                 }
2286 
2287                 r = dgraph_add_instance(inst_fmri, inst, B_FALSE);
2288                 switch (r) {
2289                 case 0:
2290                 case ECANCELED:
2291                         break;
2292 
2293                 case EEXIST:
2294                         continue;
2295 
2296                 case ECONNABORTED:
2297                         libscf_handle_rebind(h);
2298                         *reboundp = B_TRUE;
2299                         goto rebound;
2300 
2301                 case EINVAL:
2302                 default:
2303                         bad_error("dgraph_add_instance", r);
2304                 }
2305         }
2306 
2307         ret = 0;
2308 
2309 out:
2310         startd_free(inst_fmri, max_scf_fmri_size);
2311         scf_iter_destroy(iter);
2312         scf_instance_destroy(inst);
2313         scf_service_destroy(svc);
2314         return (ret);
2315 }
2316 
2317 struct depfmri_info {
2318         graph_vertex_t  *v;             /* GVT_GROUP vertex */
2319         gv_type_t       type;           /* type of dependency */
2320         const char      *inst_fmri;     /* FMRI of parental GVT_INST vert. */
2321         const char      *pg_name;       /* Name of dependency pg */
2322         scf_handle_t    *h;
2323         int             err;            /* return error code */
2324         int             **pathp;        /* return circular dependency path */
2325 };
2326 
2327 /*
2328  * Find or create a vertex for fmri and make info->v depend on it.
2329  * Returns
2330  *   0 - success
2331  *   nonzero - failure
2332  *
2333  * On failure, sets info->err to
2334  *   EINVAL - fmri is invalid
2335  *            fmri does not match info->type
2336  *   ELOOP - Adding the dependency creates a circular dependency.  *info->pathp
2337  *           will point to an array of the ids of the members of the cycle.
2338  *   ECONNABORTED - repository connection was broken
2339  *   ECONNRESET - succeeded, but repository connection was reset
2340  */
2341 static int
2342 process_dependency_fmri(const char *fmri, struct depfmri_info *info)
2343 {
2344         int err;
2345         graph_vertex_t *depgroup_v, *v;
2346         char *fmri_copy, *cfmri;
2347         size_t fmri_copy_sz;
2348         const char *scope, *service, *instance, *pg;
2349         scf_instance_t *inst;
2350         boolean_t rebound;
2351 
2352         assert(MUTEX_HELD(&dgraph_lock));
2353 
2354         /* Get or create vertex for FMRI */
2355         depgroup_v = info->v;
2356 
2357         if (strncmp(fmri, "file:", sizeof ("file:") - 1) == 0) {
2358                 if (info->type != GVT_FILE) {
2359                         log_framework(LOG_NOTICE,
2360                             "FMRI \"%s\" is not allowed for the \"%s\" "
2361                             "dependency's type of instance %s.\n", fmri,
2362                             info->pg_name, info->inst_fmri);
2363                         return (info->err = EINVAL);
2364                 }
2365 
2366                 err = graph_insert_vertex_unconfigured(fmri, info->type, 0,
2367                     RERR_NONE, &v);
2368                 switch (err) {
2369                 case 0:
2370                         break;
2371 
2372                 case EEXIST:
2373                         assert(v->gv_type == GVT_FILE);
2374                         break;
2375 
2376                 case EINVAL:            /* prevented above */
2377                 default:
2378                         bad_error("graph_insert_vertex_unconfigured", err);
2379                 }
2380         } else {
2381                 if (info->type != GVT_INST) {
2382                         log_framework(LOG_NOTICE,
2383                             "FMRI \"%s\" is not allowed for the \"%s\" "
2384                             "dependency's type of instance %s.\n", fmri,
2385                             info->pg_name, info->inst_fmri);
2386                         return (info->err = EINVAL);
2387                 }
2388 
2389                 /*
2390                  * We must canonify fmri & add a vertex for it.
2391                  */
2392                 fmri_copy_sz = strlen(fmri) + 1;
2393                 fmri_copy = startd_alloc(fmri_copy_sz);
2394                 (void) strcpy(fmri_copy, fmri);
2395 
2396                 /* Determine if the FMRI is a property group or instance */
2397                 if (scf_parse_svc_fmri(fmri_copy, &scope, &service,
2398                     &instance, &pg, NULL) != 0) {
2399                         startd_free(fmri_copy, fmri_copy_sz);
2400                         log_framework(LOG_NOTICE,
2401                             "Dependency \"%s\" of %s has invalid FMRI "
2402                             "\"%s\".\n", info->pg_name, info->inst_fmri,
2403                             fmri);
2404                         return (info->err = EINVAL);
2405                 }
2406 
2407                 if (service == NULL || pg != NULL) {
2408                         startd_free(fmri_copy, fmri_copy_sz);
2409                         log_framework(LOG_NOTICE,
2410                             "Dependency \"%s\" of %s does not designate a "
2411                             "service or instance.\n", info->pg_name,
2412                             info->inst_fmri);
2413                         return (info->err = EINVAL);
2414                 }
2415 
2416                 if (scope == NULL || strcmp(scope, SCF_SCOPE_LOCAL) == 0) {
2417                         cfmri = uu_msprintf("svc:/%s%s%s",
2418                             service, instance ? ":" : "", instance ? instance :
2419                             "");
2420                 } else {
2421                         cfmri = uu_msprintf("svc://%s/%s%s%s",
2422                             scope, service, instance ? ":" : "", instance ?
2423                             instance : "");
2424                 }
2425 
2426                 startd_free(fmri_copy, fmri_copy_sz);
2427 
2428                 err = graph_insert_vertex_unconfigured(cfmri, instance ?
2429                     GVT_INST : GVT_SVC, instance ? 0 : DEPGRP_REQUIRE_ANY,
2430                     RERR_NONE, &v);
2431                 uu_free(cfmri);
2432                 switch (err) {
2433                 case 0:
2434                         break;
2435 
2436                 case EEXIST:
2437                         /* Verify v. */
2438                         if (instance != NULL)
2439                                 assert(v->gv_type == GVT_INST);
2440                         else
2441                                 assert(v->gv_type == GVT_SVC);
2442                         break;
2443 
2444                 default:
2445                         bad_error("graph_insert_vertex_unconfigured", err);
2446                 }
2447         }
2448 
2449         /* Add dependency from depgroup_v to new vertex */
2450         info->err = graph_insert_dependency(depgroup_v, v, info->pathp);
2451         switch (info->err) {
2452         case 0:
2453                 break;
2454 
2455         case ELOOP:
2456                 return (ELOOP);
2457 
2458         default:
2459                 bad_error("graph_insert_dependency", info->err);
2460         }
2461 
2462         /* This must be after we insert the dependency, to avoid looping. */
2463         switch (v->gv_type) {
2464         case GVT_INST:
2465                 if ((v->gv_flags & GV_CONFIGURED) != 0)
2466                         break;
2467 
2468                 inst = safe_scf_instance_create(info->h);
2469 
2470                 rebound = B_FALSE;
2471 
2472 rebound:
2473                 err = libscf_lookup_instance(v->gv_name, inst);
2474                 switch (err) {
2475                 case 0:
2476                         err = configure_vertex(v, inst);
2477                         switch (err) {
2478                         case 0:
2479                         case ECANCELED:
2480                                 break;
2481 
2482                         case ECONNABORTED:
2483                                 libscf_handle_rebind(info->h);
2484                                 rebound = B_TRUE;
2485                                 goto rebound;
2486 
2487                         default:
2488                                 bad_error("configure_vertex", err);
2489                         }
2490                         break;
2491 
2492                 case ENOENT:
2493                         break;
2494 
2495                 case ECONNABORTED:
2496                         libscf_handle_rebind(info->h);
2497                         rebound = B_TRUE;
2498                         goto rebound;
2499 
2500                 case EINVAL:
2501                 case ENOTSUP:
2502                 default:
2503                         bad_error("libscf_fmri_get_instance", err);
2504                 }
2505 
2506                 scf_instance_destroy(inst);
2507 
2508                 if (rebound)
2509                         return (info->err = ECONNRESET);
2510                 break;
2511 
2512         case GVT_SVC:
2513                 (void) add_service(v->gv_name, info->h, &rebound);
2514                 if (rebound)
2515                         return (info->err = ECONNRESET);
2516         }
2517 
2518         return (0);
2519 }
2520 
2521 struct deppg_info {
2522         graph_vertex_t  *v;             /* GVT_INST vertex */
2523         int             err;            /* return error */
2524         int             **pathp;        /* return circular dependency path */
2525 };
2526 
2527 /*
2528  * Make info->v depend on a new GVT_GROUP node for this property group,
2529  * and then call process_dependency_fmri() for the values of the entity
2530  * property.  Return 0 on success, or if something goes wrong return nonzero
2531  * and set info->err to ECONNABORTED, EINVAL, or the error code returned by
2532  * process_dependency_fmri().
2533  */
2534 static int
2535 process_dependency_pg(scf_propertygroup_t *pg, struct deppg_info *info)
2536 {
2537         scf_handle_t *h;
2538         depgroup_type_t deptype;
2539         restarter_error_t rerr;
2540         struct depfmri_info linfo;
2541         char *fmri, *pg_name;
2542         size_t fmri_sz;
2543         graph_vertex_t *depgrp;
2544         scf_property_t *prop;
2545         int err;
2546         int empty;
2547         scf_error_t scferr;
2548         ssize_t len;
2549 
2550         assert(MUTEX_HELD(&dgraph_lock));
2551 
2552         h = scf_pg_handle(pg);
2553 
2554         pg_name = startd_alloc(max_scf_name_size);
2555 
2556         len = scf_pg_get_name(pg, pg_name, max_scf_name_size);
2557         if (len < 0) {
2558                 startd_free(pg_name, max_scf_name_size);
2559                 switch (scf_error()) {
2560                 case SCF_ERROR_CONNECTION_BROKEN:
2561                 default:
2562                         return (info->err = ECONNABORTED);
2563 
2564                 case SCF_ERROR_DELETED:
2565                         return (info->err = 0);
2566 
2567                 case SCF_ERROR_NOT_SET:
2568                         bad_error("scf_pg_get_name", scf_error());
2569                 }
2570         }
2571 
2572         /*
2573          * Skip over empty dependency groups.  Since dependency property
2574          * groups are updated atomically, they are either empty or
2575          * fully populated.
2576          */
2577         empty = depgroup_empty(h, pg);
2578         if (empty < 0) {
2579                 log_error(LOG_INFO,
2580                     "Error reading dependency group \"%s\" of %s: %s\n",
2581                     pg_name, info->v->gv_name, scf_strerror(scf_error()));
2582                 startd_free(pg_name, max_scf_name_size);
2583                 return (info->err = EINVAL);
2584 
2585         } else if (empty == 1) {
2586                 log_framework(LOG_DEBUG,
2587                     "Ignoring empty dependency group \"%s\" of %s\n",
2588                     pg_name, info->v->gv_name);
2589                 startd_free(pg_name, max_scf_name_size);
2590                 return (info->err = 0);
2591         }
2592 
2593         fmri_sz = strlen(info->v->gv_name) + 1 + len + 1;
2594         fmri = startd_alloc(fmri_sz);
2595 
2596         (void) snprintf(fmri, fmri_sz, "%s>%s", info->v->gv_name,
2597             pg_name);
2598 
2599         /* Validate the pg before modifying the graph */
2600         deptype = depgroup_read_grouping(h, pg);
2601         if (deptype == DEPGRP_UNSUPPORTED) {
2602                 log_error(LOG_INFO,
2603                     "Dependency \"%s\" of %s has an unknown grouping value.\n",
2604                     pg_name, info->v->gv_name);
2605                 startd_free(fmri, fmri_sz);
2606                 startd_free(pg_name, max_scf_name_size);
2607                 return (info->err = EINVAL);
2608         }
2609 
2610         rerr = depgroup_read_restart(h, pg);
2611         if (rerr == RERR_UNSUPPORTED) {
2612                 log_error(LOG_INFO,
2613                     "Dependency \"%s\" of %s has an unknown restart_on value."
2614                     "\n", pg_name, info->v->gv_name);
2615                 startd_free(fmri, fmri_sz);
2616                 startd_free(pg_name, max_scf_name_size);
2617                 return (info->err = EINVAL);
2618         }
2619 
2620         prop = safe_scf_property_create(h);
2621 
2622         if (scf_pg_get_property(pg, SCF_PROPERTY_ENTITIES, prop) != 0) {
2623                 scferr = scf_error();
2624                 scf_property_destroy(prop);
2625                 if (scferr == SCF_ERROR_DELETED) {
2626                         startd_free(fmri, fmri_sz);
2627                         startd_free(pg_name, max_scf_name_size);
2628                         return (info->err = 0);
2629                 } else if (scferr != SCF_ERROR_NOT_FOUND) {
2630                         startd_free(fmri, fmri_sz);
2631                         startd_free(pg_name, max_scf_name_size);
2632                         return (info->err = ECONNABORTED);
2633                 }
2634 
2635                 log_error(LOG_INFO,
2636                     "Dependency \"%s\" of %s is missing a \"%s\" property.\n",
2637                     pg_name, info->v->gv_name, SCF_PROPERTY_ENTITIES);
2638 
2639                 startd_free(fmri, fmri_sz);
2640                 startd_free(pg_name, max_scf_name_size);
2641 
2642                 return (info->err = EINVAL);
2643         }
2644 
2645         /* Create depgroup vertex for pg */
2646         err = graph_insert_vertex_unconfigured(fmri, GVT_GROUP, deptype,
2647             rerr, &depgrp);
2648         assert(err == 0);
2649         startd_free(fmri, fmri_sz);
2650 
2651         /* Add dependency from inst vertex to new vertex */
2652         err = graph_insert_dependency(info->v, depgrp, info->pathp);
2653         /* ELOOP can't happen because this should be a new vertex */
2654         assert(err == 0);
2655 
2656         linfo.v = depgrp;
2657         linfo.type = depgroup_read_scheme(h, pg);
2658         linfo.inst_fmri = info->v->gv_name;
2659         linfo.pg_name = pg_name;
2660         linfo.h = h;
2661         linfo.err = 0;
2662         linfo.pathp = info->pathp;
2663         err = walk_property_astrings(prop, (callback_t)process_dependency_fmri,
2664             &linfo);
2665 
2666         scf_property_destroy(prop);
2667         startd_free(pg_name, max_scf_name_size);
2668 
2669         switch (err) {
2670         case 0:
2671         case EINTR:
2672                 return (info->err = linfo.err);
2673 
2674         case ECONNABORTED:
2675         case EINVAL:
2676                 return (info->err = err);
2677 
2678         case ECANCELED:
2679                 return (info->err = 0);
2680 
2681         case ECONNRESET:
2682                 return (info->err = ECONNABORTED);
2683 
2684         default:
2685                 bad_error("walk_property_astrings", err);
2686                 /* NOTREACHED */
2687         }
2688 }
2689 
2690 /*
2691  * Build the dependency info for v from the repository.  Returns 0 on success,
2692  * ECONNABORTED on repository disconnection, EINVAL if the repository
2693  * configuration is invalid, and ELOOP if a dependency would cause a cycle.
2694  * In the last case, *pathp will point to a -1-terminated array of ids which
2695  * constitute the rest of the dependency cycle.
2696  */
2697 static int
2698 set_dependencies(graph_vertex_t *v, scf_instance_t *inst, int **pathp)
2699 {
2700         struct deppg_info info;
2701         int err;
2702         uint_t old_configured;
2703 
2704         assert(MUTEX_HELD(&dgraph_lock));
2705 
2706         /*
2707          * Mark the vertex as configured during dependency insertion to avoid
2708          * dependency cycles (which can appear in the graph if one of the
2709          * vertices is an exclusion-group).
2710          */
2711         old_configured = v->gv_flags & GV_CONFIGURED;
2712         v->gv_flags |= GV_CONFIGURED;
2713 
2714         info.err = 0;
2715         info.v = v;
2716         info.pathp = pathp;
2717 
2718         err = walk_dependency_pgs(inst, (callback_t)process_dependency_pg,
2719             &info);
2720 
2721         if (!old_configured)
2722                 v->gv_flags &= ~GV_CONFIGURED;
2723 
2724         switch (err) {
2725         case 0:
2726         case EINTR:
2727                 return (info.err);
2728 
2729         case ECONNABORTED:
2730                 return (ECONNABORTED);
2731 
2732         case ECANCELED:
2733                 /* Should get delete event, so return 0. */
2734                 return (0);
2735 
2736         default:
2737                 bad_error("walk_dependency_pgs", err);
2738                 /* NOTREACHED */
2739         }
2740 }
2741 
2742 
2743 static void
2744 handle_cycle(const char *fmri, int *path)
2745 {
2746         const char *cp;
2747         size_t sz;
2748 
2749         assert(MUTEX_HELD(&dgraph_lock));
2750 
2751         path_to_str(path, (char **)&cp, &sz);
2752 
2753         log_error(LOG_ERR, "Transitioning %s to maintenance "
2754             "because it completes a dependency cycle (see svcs -xv for "
2755             "details):\n%s", fmri ? fmri : "?", cp);
2756 
2757         startd_free((void *)cp, sz);
2758 }
2759 
2760 /*
2761  * Increment the vertex's reference count to prevent the vertex removal
2762  * from the dgraph.
2763  */
2764 static void
2765 vertex_ref(graph_vertex_t *v)
2766 {
2767         assert(MUTEX_HELD(&dgraph_lock));
2768 
2769         v->gv_refs++;
2770 }
2771 
2772 /*
2773  * Decrement the vertex's reference count and remove the vertex from
2774  * the dgraph when possible.
2775  *
2776  * Return VERTEX_REMOVED when the vertex has been removed otherwise
2777  * return VERTEX_INUSE.
2778  */
2779 static int
2780 vertex_unref(graph_vertex_t *v)
2781 {
2782         assert(MUTEX_HELD(&dgraph_lock));
2783         assert(v->gv_refs > 0);
2784 
2785         v->gv_refs--;
2786 
2787         return (free_if_unrefed(v));
2788 }
2789 
2790 /*
2791  * When run on the dependencies of a vertex, populates list with
2792  * graph_edge_t's which point to the service vertices or the instance
2793  * vertices (no GVT_GROUP nodes) on which the vertex depends.
2794  *
2795  * Increment the vertex's reference count once the vertex is inserted
2796  * in the list. The vertex won't be able to be deleted from the dgraph
2797  * while it is referenced.
2798  */
2799 static int
2800 append_svcs_or_insts(graph_edge_t *e, uu_list_t *list)
2801 {
2802         graph_vertex_t *v = e->ge_vertex;
2803         graph_edge_t *new;
2804         int r;
2805 
2806         switch (v->gv_type) {
2807         case GVT_INST:
2808         case GVT_SVC:
2809                 break;
2810 
2811         case GVT_GROUP:
2812                 r = uu_list_walk(v->gv_dependencies,
2813                     (uu_walk_fn_t *)append_svcs_or_insts, list, 0);
2814                 assert(r == 0);
2815                 return (UU_WALK_NEXT);
2816 
2817         case GVT_FILE:
2818                 return (UU_WALK_NEXT);
2819 
2820         default:
2821 #ifndef NDEBUG
2822                 uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
2823                     __LINE__, v->gv_type);
2824 #endif
2825                 abort();
2826         }
2827 
2828         new = startd_alloc(sizeof (*new));
2829         new->ge_vertex = v;
2830         uu_list_node_init(new, &new->ge_link, graph_edge_pool);
2831         r = uu_list_insert_before(list, NULL, new);
2832         assert(r == 0);
2833 
2834         /*
2835          * Because we are inserting the vertex in a list, we don't want
2836          * the vertex to be freed while the list is in use. In order to
2837          * achieve that, increment the vertex's reference count.
2838          */
2839         vertex_ref(v);
2840 
2841         return (UU_WALK_NEXT);
2842 }
2843 
2844 static boolean_t
2845 should_be_in_subgraph(graph_vertex_t *v)
2846 {
2847         graph_edge_t *e;
2848 
2849         if (v == milestone)
2850                 return (B_TRUE);
2851 
2852         /*
2853          * v is in the subgraph if any of its dependents are in the subgraph.
2854          * Except for EXCLUDE_ALL dependents.  And OPTIONAL dependents only
2855          * count if we're enabled.
2856          */
2857         for (e = uu_list_first(v->gv_dependents);
2858             e != NULL;
2859             e = uu_list_next(v->gv_dependents, e)) {
2860                 graph_vertex_t *dv = e->ge_vertex;
2861 
2862                 if (!(dv->gv_flags & GV_INSUBGRAPH))
2863                         continue;
2864 
2865                 /*
2866                  * Don't include instances that are optional and disabled.
2867                  */
2868                 if (v->gv_type == GVT_INST && dv->gv_type == GVT_SVC) {
2869 
2870                         int in = 0;
2871                         graph_edge_t *ee;
2872 
2873                         for (ee = uu_list_first(dv->gv_dependents);
2874                             ee != NULL;
2875                             ee = uu_list_next(dv->gv_dependents, ee)) {
2876 
2877                                 graph_vertex_t *ddv = e->ge_vertex;
2878 
2879                                 if (ddv->gv_type == GVT_GROUP &&
2880                                     ddv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2881                                         continue;
2882 
2883                                 if (ddv->gv_type == GVT_GROUP &&
2884                                     ddv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2885                                     !(v->gv_flags & GV_ENBLD_NOOVR))
2886                                         continue;
2887 
2888                                 in = 1;
2889                         }
2890                         if (!in)
2891                                 continue;
2892                 }
2893                 if (v->gv_type == GVT_INST &&
2894                     dv->gv_type == GVT_GROUP &&
2895                     dv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2896                     !(v->gv_flags & GV_ENBLD_NOOVR))
2897                         continue;
2898 
2899                 /* Don't include excluded services and instances */
2900                 if (dv->gv_type == GVT_GROUP &&
2901                     dv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2902                         continue;
2903 
2904                 return (B_TRUE);
2905         }
2906 
2907         return (B_FALSE);
2908 }
2909 
2910 /*
2911  * Ensures that GV_INSUBGRAPH is set properly for v and its descendents.  If
2912  * any bits change, manipulate the repository appropriately.  Returns 0 or
2913  * ECONNABORTED.
2914  */
2915 static int
2916 eval_subgraph(graph_vertex_t *v, scf_handle_t *h)
2917 {
2918         boolean_t old = (v->gv_flags & GV_INSUBGRAPH) != 0;
2919         boolean_t new;
2920         graph_edge_t *e;
2921         scf_instance_t *inst;
2922         int ret = 0, r;
2923 
2924         assert(milestone != NULL && milestone != MILESTONE_NONE);
2925 
2926         new = should_be_in_subgraph(v);
2927 
2928         if (new == old)
2929                 return (0);
2930 
2931         log_framework(LOG_DEBUG, new ? "Adding %s to the subgraph.\n" :
2932             "Removing %s from the subgraph.\n", v->gv_name);
2933 
2934         v->gv_flags = (v->gv_flags & ~GV_INSUBGRAPH) |
2935             (new ? GV_INSUBGRAPH : 0);
2936 
2937         if (v->gv_type == GVT_INST && (v->gv_flags & GV_CONFIGURED)) {
2938                 int err;
2939 
2940 get_inst:
2941                 err = libscf_fmri_get_instance(h, v->gv_name, &inst);
2942                 if (err != 0) {
2943                         switch (err) {
2944                         case ECONNABORTED:
2945                                 libscf_handle_rebind(h);
2946                                 ret = ECONNABORTED;
2947                                 goto get_inst;
2948 
2949                         case ENOENT:
2950                                 break;
2951 
2952                         case EINVAL:
2953                         case ENOTSUP:
2954                         default:
2955                                 bad_error("libscf_fmri_get_instance", err);
2956                         }
2957                 } else {
2958                         const char *f;
2959 
2960                         if (new) {
2961                                 err = libscf_delete_enable_ovr(inst);
2962                                 f = "libscf_delete_enable_ovr";
2963                         } else {
2964                                 err = libscf_set_enable_ovr(inst, 0);
2965                                 f = "libscf_set_enable_ovr";
2966                         }
2967                         scf_instance_destroy(inst);
2968                         switch (err) {
2969                         case 0:
2970                         case ECANCELED:
2971                                 break;
2972 
2973                         case ECONNABORTED:
2974                                 libscf_handle_rebind(h);
2975                                 /*
2976                                  * We must continue so the graph is updated,
2977                                  * but we must return ECONNABORTED so any
2978                                  * libscf state held by any callers is reset.
2979                                  */
2980                                 ret = ECONNABORTED;
2981                                 goto get_inst;
2982 
2983                         case EROFS:
2984                         case EPERM:
2985                                 log_error(LOG_WARNING,
2986                                     "Could not set %s/%s for %s: %s.\n",
2987                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
2988                                     v->gv_name, strerror(err));
2989                                 break;
2990 
2991                         default:
2992                                 bad_error(f, err);
2993                         }
2994                 }
2995         }
2996 
2997         for (e = uu_list_first(v->gv_dependencies);
2998             e != NULL;
2999             e = uu_list_next(v->gv_dependencies, e)) {
3000                 r = eval_subgraph(e->ge_vertex, h);
3001                 if (r != 0) {
3002                         assert(r == ECONNABORTED);
3003                         ret = ECONNABORTED;
3004                 }
3005         }
3006 
3007         return (ret);
3008 }
3009 
3010 /*
3011  * Delete the (property group) dependencies of v & create new ones based on
3012  * inst.  If doing so would create a cycle, log a message and put the instance
3013  * into maintenance.  Update GV_INSUBGRAPH flags as necessary.  Returns 0 or
3014  * ECONNABORTED.
3015  */
3016 int
3017 refresh_vertex(graph_vertex_t *v, scf_instance_t *inst)
3018 {
3019         int err;
3020         int *path;
3021         char *fmri;
3022         int r;
3023         scf_handle_t *h = scf_instance_handle(inst);
3024         uu_list_t *old_deps;
3025         int ret = 0;
3026         graph_edge_t *e;
3027         graph_vertex_t *vv;
3028 
3029         assert(MUTEX_HELD(&dgraph_lock));
3030         assert(v->gv_type == GVT_INST);
3031 
3032         log_framework(LOG_DEBUG, "Graph engine: Refreshing %s.\n", v->gv_name);
3033 
3034         if (milestone > MILESTONE_NONE) {
3035                 /*
3036                  * In case some of v's dependencies are being deleted we must
3037                  * make a list of them now for GV_INSUBGRAPH-flag evaluation
3038                  * after the new dependencies are in place.
3039                  */
3040                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
3041 
3042                 err = uu_list_walk(v->gv_dependencies,
3043                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
3044                 assert(err == 0);
3045         }
3046 
3047         delete_instance_dependencies(v, B_FALSE);
3048 
3049         err = set_dependencies(v, inst, &path);
3050         switch (err) {
3051         case 0:
3052                 break;
3053 
3054         case ECONNABORTED:
3055                 ret = err;
3056                 goto out;
3057 
3058         case EINVAL:
3059         case ELOOP:
3060                 r = libscf_instance_get_fmri(inst, &fmri);
3061                 switch (r) {
3062                 case 0:
3063                         break;
3064 
3065                 case ECONNABORTED:
3066                         ret = ECONNABORTED;
3067                         goto out;
3068 
3069                 case ECANCELED:
3070                         ret = 0;
3071                         goto out;
3072 
3073                 default:
3074                         bad_error("libscf_instance_get_fmri", r);
3075                 }
3076 
3077                 if (err == EINVAL) {
3078                         log_error(LOG_ERR, "Transitioning %s "
3079                             "to maintenance due to misconfiguration.\n",
3080                             fmri ? fmri : "?");
3081                         vertex_send_event(v,
3082                             RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY);
3083                 } else {
3084                         handle_cycle(fmri, path);
3085                         vertex_send_event(v,
3086                             RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE);
3087                 }
3088                 startd_free(fmri, max_scf_fmri_size);
3089                 ret = 0;
3090                 goto out;
3091 
3092         default:
3093                 bad_error("set_dependencies", err);
3094         }
3095 
3096         if (milestone > MILESTONE_NONE) {
3097                 boolean_t aborted = B_FALSE;
3098 
3099                 for (e = uu_list_first(old_deps);
3100                     e != NULL;
3101                     e = uu_list_next(old_deps, e)) {
3102                         vv = e->ge_vertex;
3103 
3104                         if (vertex_unref(vv) == VERTEX_INUSE &&
3105                             eval_subgraph(vv, h) == ECONNABORTED)
3106                                 aborted = B_TRUE;
3107                 }
3108 
3109                 for (e = uu_list_first(v->gv_dependencies);
3110                     e != NULL;
3111                     e = uu_list_next(v->gv_dependencies, e)) {
3112                         if (eval_subgraph(e->ge_vertex, h) ==
3113                             ECONNABORTED)
3114                                 aborted = B_TRUE;
3115                 }
3116 
3117                 if (aborted) {
3118                         ret = ECONNABORTED;
3119                         goto out;
3120                 }
3121         }
3122 
3123         graph_start_if_satisfied(v);
3124 
3125         ret = 0;
3126 
3127 out:
3128         if (milestone > MILESTONE_NONE) {
3129                 void *cookie = NULL;
3130 
3131                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL)
3132                         startd_free(e, sizeof (*e));
3133 
3134                 uu_list_destroy(old_deps);
3135         }
3136 
3137         return (ret);
3138 }
3139 
3140 /*
3141  * Set up v according to inst.  That is, make sure it depends on its
3142  * restarter and set up its dependencies.  Send the ADD_INSTANCE command to
3143  * the restarter, and send ENABLE or DISABLE as appropriate.
3144  *
3145  * Returns 0 on success, ECONNABORTED on repository disconnection, or
3146  * ECANCELED if inst is deleted.
3147  */
3148 static int
3149 configure_vertex(graph_vertex_t *v, scf_instance_t *inst)
3150 {
3151         scf_handle_t *h;
3152         scf_propertygroup_t *pg;
3153         scf_snapshot_t *snap;
3154         char *restarter_fmri = startd_alloc(max_scf_value_size);
3155         int enabled, enabled_ovr;
3156         int err;
3157         int *path;
3158         int deathrow;
3159         int32_t tset;
3160 
3161         restarter_fmri[0] = '\0';
3162 
3163         assert(MUTEX_HELD(&dgraph_lock));
3164         assert(v->gv_type == GVT_INST);
3165         assert((v->gv_flags & GV_CONFIGURED) == 0);
3166 
3167         /* GV_INSUBGRAPH should already be set properly. */
3168         assert(should_be_in_subgraph(v) ==
3169             ((v->gv_flags & GV_INSUBGRAPH) != 0));
3170 
3171         /*
3172          * If the instance fmri is in the deathrow list then set the
3173          * GV_DEATHROW flag on the vertex and create and set to true the
3174          * SCF_PROPERTY_DEATHROW boolean property in the non-persistent
3175          * repository for this instance fmri.
3176          */
3177         if ((v->gv_flags & GV_DEATHROW) ||
3178             (is_fmri_in_deathrow(v->gv_name) == B_TRUE)) {
3179                 if ((v->gv_flags & GV_DEATHROW) == 0) {
3180                         /*
3181                          * Set flag GV_DEATHROW, create and set to true
3182                          * the SCF_PROPERTY_DEATHROW property in the
3183                          * non-persistent repository for this instance fmri.
3184                          */
3185                         v->gv_flags |= GV_DEATHROW;
3186 
3187                         switch (err = libscf_set_deathrow(inst, 1)) {
3188                         case 0:
3189                                 break;
3190 
3191                         case ECONNABORTED:
3192                         case ECANCELED:
3193                                 startd_free(restarter_fmri, max_scf_value_size);
3194                                 return (err);
3195 
3196                         case EROFS:
3197                                 log_error(LOG_WARNING, "Could not set %s/%s "
3198                                     "for deathrow %s: %s.\n",
3199                                     SCF_PG_DEATHROW, SCF_PROPERTY_DEATHROW,
3200                                     v->gv_name, strerror(err));
3201                                 break;
3202 
3203                         case EPERM:
3204                                 uu_die("Permission denied.\n");
3205                                 /* NOTREACHED */
3206 
3207                         default:
3208                                 bad_error("libscf_set_deathrow", err);
3209                         }
3210                         log_framework(LOG_DEBUG, "Deathrow, graph set %s.\n",
3211                             v->gv_name);
3212                 }
3213                 startd_free(restarter_fmri, max_scf_value_size);
3214                 return (0);
3215         }
3216 
3217         h = scf_instance_handle(inst);
3218 
3219         /*
3220          * Using a temporary deathrow boolean property, set through
3221          * libscf_set_deathrow(), only for fmris on deathrow, is necessary
3222          * because deathrow_fini() may already have been called, and in case
3223          * of a refresh, GV_DEATHROW may need to be set again.
3224          * libscf_get_deathrow() sets deathrow to 1 only if this instance
3225          * has a temporary boolean property named 'deathrow' valued true
3226          * in a property group 'deathrow', -1 or 0 in all other cases.
3227          */
3228         err = libscf_get_deathrow(h, inst, &deathrow);
3229         switch (err) {
3230         case 0:
3231                 break;
3232 
3233         case ECONNABORTED:
3234         case ECANCELED:
3235                 startd_free(restarter_fmri, max_scf_value_size);
3236                 return (err);
3237 
3238         default:
3239                 bad_error("libscf_get_deathrow", err);
3240         }
3241 
3242         if (deathrow == 1) {
3243                 v->gv_flags |= GV_DEATHROW;
3244                 startd_free(restarter_fmri, max_scf_value_size);
3245                 return (0);
3246         }
3247 
3248         log_framework(LOG_DEBUG, "Graph adding %s.\n", v->gv_name);
3249 
3250         /*
3251          * If the instance does not have a restarter property group,
3252          * initialize its state to uninitialized/none, in case the restarter
3253          * is not enabled.
3254          */
3255         pg = safe_scf_pg_create(h);
3256 
3257         if (scf_instance_get_pg(inst, SCF_PG_RESTARTER, pg) != 0) {
3258                 instance_data_t idata;
3259                 uint_t count = 0, msecs = ALLOC_DELAY;
3260 
3261                 switch (scf_error()) {
3262                 case SCF_ERROR_NOT_FOUND:
3263                         break;
3264 
3265                 case SCF_ERROR_CONNECTION_BROKEN:
3266                 default:
3267                         scf_pg_destroy(pg);
3268                         startd_free(restarter_fmri, max_scf_value_size);
3269                         return (ECONNABORTED);
3270 
3271                 case SCF_ERROR_DELETED:
3272                         scf_pg_destroy(pg);
3273                         startd_free(restarter_fmri, max_scf_value_size);
3274                         return (ECANCELED);
3275 
3276                 case SCF_ERROR_NOT_SET:
3277                         bad_error("scf_instance_get_pg", scf_error());
3278                 }
3279 
3280                 switch (err = libscf_instance_get_fmri(inst,
3281                     (char **)&idata.i_fmri)) {
3282                 case 0:
3283                         break;
3284 
3285                 case ECONNABORTED:
3286                 case ECANCELED:
3287                         scf_pg_destroy(pg);
3288                         startd_free(restarter_fmri, max_scf_value_size);
3289                         return (err);
3290 
3291                 default:
3292                         bad_error("libscf_instance_get_fmri", err);
3293                 }
3294 
3295                 idata.i_state = RESTARTER_STATE_NONE;
3296                 idata.i_next_state = RESTARTER_STATE_NONE;
3297 
3298 init_state:
3299                 switch (err = _restarter_commit_states(h, &idata,
3300                     RESTARTER_STATE_UNINIT, RESTARTER_STATE_NONE,
3301                     restarter_get_str_short(restarter_str_insert_in_graph))) {
3302                 case 0:
3303                         break;
3304 
3305                 case ENOMEM:
3306                         ++count;
3307                         if (count < ALLOC_RETRY) {
3308                                 (void) poll(NULL, 0, msecs);
3309                                 msecs *= ALLOC_DELAY_MULT;
3310                                 goto init_state;
3311                         }
3312 
3313                         uu_die("Insufficient memory.\n");
3314                         /* NOTREACHED */
3315 
3316                 case ECONNABORTED:
3317                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3318                         scf_pg_destroy(pg);
3319                         startd_free(restarter_fmri, max_scf_value_size);
3320                         return (ECONNABORTED);
3321 
3322                 case ENOENT:
3323                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3324                         scf_pg_destroy(pg);
3325                         startd_free(restarter_fmri, max_scf_value_size);
3326                         return (ECANCELED);
3327 
3328                 case EPERM:
3329                 case EACCES:
3330                 case EROFS:
3331                         log_error(LOG_NOTICE, "Could not initialize state for "
3332                             "%s: %s.\n", idata.i_fmri, strerror(err));
3333                         break;
3334 
3335                 case EINVAL:
3336                 default:
3337                         bad_error("_restarter_commit_states", err);
3338                 }
3339 
3340                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3341         }
3342 
3343         scf_pg_destroy(pg);
3344 
3345         if (milestone != NULL) {
3346                 /*
3347                  * Make sure the enable-override is set properly before we
3348                  * read whether we should be enabled.
3349                  */
3350                 if (milestone == MILESTONE_NONE ||
3351                     !(v->gv_flags & GV_INSUBGRAPH)) {
3352                         /*
3353                          * This might seem unjustified after the milestone
3354                          * transition has completed (non_subgraph_svcs == 0),
3355                          * but it's important because when we boot to
3356                          * a milestone, we set the milestone before populating
3357                          * the graph, and all of the new non-subgraph services
3358                          * need to be disabled here.
3359                          */
3360                         switch (err = libscf_set_enable_ovr(inst, 0)) {
3361                         case 0:
3362                                 break;
3363 
3364                         case ECONNABORTED:
3365                         case ECANCELED:
3366                                 startd_free(restarter_fmri, max_scf_value_size);
3367                                 return (err);
3368 
3369                         case EROFS:
3370                                 log_error(LOG_WARNING,
3371                                     "Could not set %s/%s for %s: %s.\n",
3372                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
3373                                     v->gv_name, strerror(err));
3374                                 break;
3375 
3376                         case EPERM:
3377                                 uu_die("Permission denied.\n");
3378                                 /* NOTREACHED */
3379 
3380                         default:
3381                                 bad_error("libscf_set_enable_ovr", err);
3382                         }
3383                 } else {
3384                         assert(v->gv_flags & GV_INSUBGRAPH);
3385                         switch (err = libscf_delete_enable_ovr(inst)) {
3386                         case 0:
3387                                 break;
3388 
3389                         case ECONNABORTED:
3390                         case ECANCELED:
3391                                 startd_free(restarter_fmri, max_scf_value_size);
3392                                 return (err);
3393 
3394                         case EPERM:
3395                                 uu_die("Permission denied.\n");
3396                                 /* NOTREACHED */
3397 
3398                         default:
3399                                 bad_error("libscf_delete_enable_ovr", err);
3400                         }
3401                 }
3402         }
3403 
3404         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
3405             &enabled_ovr, &restarter_fmri);
3406         switch (err) {
3407         case 0:
3408                 break;
3409 
3410         case ECONNABORTED:
3411         case ECANCELED:
3412                 startd_free(restarter_fmri, max_scf_value_size);
3413                 return (err);
3414 
3415         case ENOENT:
3416                 log_framework(LOG_DEBUG,
3417                     "Ignoring %s because it has no general property group.\n",
3418                     v->gv_name);
3419                 startd_free(restarter_fmri, max_scf_value_size);
3420                 return (0);
3421 
3422         default:
3423                 bad_error("libscf_get_basic_instance_data", err);
3424         }
3425 
3426         if ((tset = libscf_get_stn_tset(inst)) == -1) {
3427                 log_framework(LOG_WARNING,
3428                     "Failed to get notification parameters for %s: %s\n",
3429                     v->gv_name, scf_strerror(scf_error()));
3430                 v->gv_stn_tset = 0;
3431         } else {
3432                 v->gv_stn_tset = tset;
3433         }
3434         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
3435                 stn_global = v->gv_stn_tset;
3436 
3437         if (enabled == -1) {
3438                 startd_free(restarter_fmri, max_scf_value_size);
3439                 return (0);
3440         }
3441 
3442         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
3443             (enabled ? GV_ENBLD_NOOVR : 0);
3444 
3445         if (enabled_ovr != -1)
3446                 enabled = enabled_ovr;
3447 
3448         v->gv_state = RESTARTER_STATE_UNINIT;
3449 
3450         snap = libscf_get_or_make_running_snapshot(inst, v->gv_name, B_TRUE);
3451         scf_snapshot_destroy(snap);
3452 
3453         /* Set up the restarter. (Sends _ADD_INSTANCE on success.) */
3454         err = graph_change_restarter(v, restarter_fmri, h, &path);
3455         if (err != 0) {
3456                 instance_data_t idata;
3457                 uint_t count = 0, msecs = ALLOC_DELAY;
3458                 restarter_str_t reason;
3459 
3460                 if (err == ECONNABORTED) {
3461                         startd_free(restarter_fmri, max_scf_value_size);
3462                         return (err);
3463                 }
3464 
3465                 assert(err == EINVAL || err == ELOOP);
3466 
3467                 if (err == EINVAL) {
3468                         log_framework(LOG_ERR, emsg_invalid_restarter,
3469                             v->gv_name, restarter_fmri);
3470                         reason = restarter_str_invalid_restarter;
3471                 } else {
3472                         handle_cycle(v->gv_name, path);
3473                         reason = restarter_str_dependency_cycle;
3474                 }
3475 
3476                 startd_free(restarter_fmri, max_scf_value_size);
3477 
3478                 /*
3479                  * We didn't register the instance with the restarter, so we
3480                  * must set maintenance mode ourselves.
3481                  */
3482                 err = libscf_instance_get_fmri(inst, (char **)&idata.i_fmri);
3483                 if (err != 0) {
3484                         assert(err == ECONNABORTED || err == ECANCELED);
3485                         return (err);
3486                 }
3487 
3488                 idata.i_state = RESTARTER_STATE_NONE;
3489                 idata.i_next_state = RESTARTER_STATE_NONE;
3490 
3491 set_maint:
3492                 switch (err = _restarter_commit_states(h, &idata,
3493                     RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE,
3494                     restarter_get_str_short(reason))) {
3495                 case 0:
3496                         break;
3497 
3498                 case ENOMEM:
3499                         ++count;
3500                         if (count < ALLOC_RETRY) {
3501                                 (void) poll(NULL, 0, msecs);
3502                                 msecs *= ALLOC_DELAY_MULT;
3503                                 goto set_maint;
3504                         }
3505 
3506                         uu_die("Insufficient memory.\n");
3507                         /* NOTREACHED */
3508 
3509                 case ECONNABORTED:
3510                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3511                         return (ECONNABORTED);
3512 
3513                 case ENOENT:
3514                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3515                         return (ECANCELED);
3516 
3517                 case EPERM:
3518                 case EACCES:
3519                 case EROFS:
3520                         log_error(LOG_NOTICE, "Could not initialize state for "
3521                             "%s: %s.\n", idata.i_fmri, strerror(err));
3522                         break;
3523 
3524                 case EINVAL:
3525                 default:
3526                         bad_error("_restarter_commit_states", err);
3527                 }
3528 
3529                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3530 
3531                 v->gv_state = RESTARTER_STATE_MAINT;
3532 
3533                 goto out;
3534         }
3535         startd_free(restarter_fmri, max_scf_value_size);
3536 
3537         /* Add all the other dependencies. */
3538         err = refresh_vertex(v, inst);
3539         if (err != 0) {
3540                 assert(err == ECONNABORTED);
3541                 return (err);
3542         }
3543 
3544 out:
3545         v->gv_flags |= GV_CONFIGURED;
3546 
3547         graph_enable_by_vertex(v, enabled, 0);
3548 
3549         return (0);
3550 }
3551 
3552 
3553 static void
3554 kill_user_procs(void)
3555 {
3556         (void) fputs("svc.startd: Killing user processes.\n", stdout);
3557 
3558         /*
3559          * Despite its name, killall's role is to get select user processes--
3560          * basically those representing terminal-based logins-- to die.  Victims
3561          * are located by killall in the utmp database.  Since these are most
3562          * often shell based logins, and many shells mask SIGTERM (but are
3563          * responsive to SIGHUP) we first HUP and then shortly thereafter
3564          * kill -9.
3565          */
3566         (void) fork_with_timeout("/usr/sbin/killall HUP", 1, 5);
3567         (void) fork_with_timeout("/usr/sbin/killall KILL", 1, 5);
3568 
3569         /*
3570          * Note the selection of user id's 0, 1 and 15, subsequently
3571          * inverted by -v.  15 is reserved for dladmd.  Yes, this is a
3572          * kludge-- a better policy is needed.
3573          *
3574          * Note that fork_with_timeout will only wait out the 1 second
3575          * "grace time" if pkill actually returns 0.  So if there are
3576          * no matches, this will run to completion much more quickly.
3577          */
3578         (void) fork_with_timeout("/usr/bin/pkill -TERM -v -u 0,1,15", 1, 5);
3579         (void) fork_with_timeout("/usr/bin/pkill -KILL -v -u 0,1,15", 1, 5);
3580 }
3581 
3582 static void
3583 do_uadmin(void)
3584 {
3585         const char * const resetting = "/etc/svc/volatile/resetting";
3586         int fd;
3587         struct statvfs vfs;
3588         time_t now;
3589         struct tm nowtm;
3590         char down_buf[256], time_buf[256];
3591         uintptr_t mdep;
3592 #if defined(__i386)
3593         grub_boot_args_t fbarg;
3594 #endif  /* __i386 */
3595 
3596         mdep = NULL;
3597         fd = creat(resetting, 0777);
3598         if (fd >= 0)
3599                 startd_close(fd);
3600         else
3601                 uu_warn("Could not create \"%s\"", resetting);
3602 
3603         /* Kill dhcpagent if we're not using nfs for root */
3604         if ((statvfs("/", &vfs) == 0) &&
3605             (strncmp(vfs.f_basetype, "nfs", sizeof ("nfs") - 1) != 0))
3606                 fork_with_timeout("/usr/bin/pkill -x -u 0 dhcpagent", 0, 5);
3607 
3608         /*
3609          * Call sync(2) now, before we kill off user processes.  This takes
3610          * advantage of the several seconds of pause we have before the
3611          * killalls are done.  Time we can make good use of to get pages
3612          * moving out to disk.
3613          *
3614          * Inside non-global zones, we don't bother, and it's better not to
3615          * anyway, since sync(2) can have system-wide impact.
3616          */
3617         if (getzoneid() == 0)
3618                 sync();
3619 
3620         kill_user_procs();
3621 
3622         /*
3623          * Note that this must come after the killing of user procs, since
3624          * killall relies on utmpx, and this command affects the contents of
3625          * said file.
3626          */
3627         if (access("/usr/lib/acct/closewtmp", X_OK) == 0)
3628                 fork_with_timeout("/usr/lib/acct/closewtmp", 0, 5);
3629 
3630         /*
3631          * For patches which may be installed as the system is shutting
3632          * down, we need to ensure, one more time, that the boot archive
3633          * really is up to date.
3634          */
3635         if (getzoneid() == 0 && access("/usr/sbin/bootadm", X_OK) == 0)
3636                 fork_with_timeout("/usr/sbin/bootadm -ea update_all", 0, 3600);
3637 
3638         /*
3639          * Right now, fast reboot is supported only on i386.
3640          * scf_is_fastboot_default() should take care of it.
3641          * If somehow we got there on unsupported platform -
3642          * print warning and fall back to regular reboot.
3643          */
3644         if (halting == AD_FASTREBOOT) {
3645 #if defined(__i386)
3646                 int rc;
3647 
3648                 if ((rc = grub_get_boot_args(&fbarg, NULL,
3649                     GRUB_ENTRY_DEFAULT)) == 0) {
3650                         mdep = (uintptr_t)&fbarg.gba_bootargs;
3651                 } else {
3652                         /*
3653                          * Failed to read GRUB menu, fall back to normal reboot
3654                          */
3655                         halting = AD_BOOT;
3656                         uu_warn("Failed to process GRUB menu entry "
3657                             "for fast reboot.\n\t%s\n"
3658                             "Falling back to regular reboot.\n",
3659                             grub_strerror(rc));
3660                 }
3661 #else   /* __i386 */
3662                 halting = AD_BOOT;
3663                 uu_warn("Fast reboot configured, but not supported by "
3664                     "this ISA\n");
3665 #endif  /* __i386 */
3666         }
3667 
3668         fork_with_timeout("/sbin/umountall -l", 0, 5);
3669         fork_with_timeout("/sbin/umount /tmp /var/adm /var/run /var "
3670             ">/dev/null 2>&1", 0, 5);
3671 
3672         /*
3673          * Try to get to consistency for whatever UFS filesystems are left.
3674          * This is pretty expensive, so we save it for the end in the hopes of
3675          * minimizing what it must do.  The other option would be to start in
3676          * parallel with the killall's, but lockfs tends to throw out much more
3677          * than is needed, and so subsequent commands (like umountall) take a
3678          * long time to get going again.
3679          *
3680          * Inside of zones, we don't bother, since we're not about to terminate
3681          * the whole OS instance.
3682          *
3683          * On systems using only ZFS, this call to lockfs -fa is a no-op.
3684          */
3685         if (getzoneid() == 0) {
3686                 if (access("/usr/sbin/lockfs", X_OK) == 0)
3687                         fork_with_timeout("/usr/sbin/lockfs -fa", 0, 30);
3688 
3689                 sync(); /* once more, with feeling */
3690         }
3691 
3692         fork_with_timeout("/sbin/umount /usr >/dev/null 2>&1", 0, 5);
3693 
3694         /*
3695          * Construct and emit the last words from userland:
3696          * "<timestamp> The system is down.  Shutdown took <N> seconds."
3697          *
3698          * Normally we'd use syslog, but with /var and other things
3699          * potentially gone, try to minimize the external dependencies.
3700          */
3701         now = time(NULL);
3702         (void) localtime_r(&now, &nowtm);
3703 
3704         if (strftime(down_buf, sizeof (down_buf),
3705             "%b %e %T The system is down.", &nowtm) == 0) {
3706                 (void) strlcpy(down_buf, "The system is down.",
3707                     sizeof (down_buf));
3708         }
3709 
3710         if (halting_time != 0 && halting_time <= now) {
3711                 (void) snprintf(time_buf, sizeof (time_buf),
3712                     "  Shutdown took %lu seconds.", now - halting_time);
3713         } else {
3714                 time_buf[0] = '\0';
3715         }
3716         (void) printf("%s%s\n", down_buf, time_buf);
3717 
3718         (void) uadmin(A_SHUTDOWN, halting, mdep);
3719         uu_warn("uadmin() failed");
3720 
3721 #if defined(__i386)
3722         /* uadmin fail, cleanup grub_boot_args */
3723         if (halting == AD_FASTREBOOT)
3724                 grub_cleanup_boot_args(&fbarg);
3725 #endif  /* __i386 */
3726 
3727         if (remove(resetting) != 0 && errno != ENOENT)
3728                 uu_warn("Could not remove \"%s\"", resetting);
3729 }
3730 
3731 /*
3732  * If any of the up_svcs[] are online or satisfiable, return true.  If they are
3733  * all missing, disabled, in maintenance, or unsatisfiable, return false.
3734  */
3735 boolean_t
3736 can_come_up(void)
3737 {
3738         int i;
3739 
3740         assert(MUTEX_HELD(&dgraph_lock));
3741 
3742         /*
3743          * If we are booting to single user (boot -s),
3744          * SCF_MILESTONE_SINGLE_USER is needed to come up because startd
3745          * spawns sulogin after single-user is online (see specials.c).
3746          */
3747         i = (booting_to_single_user ? 0 : 1);
3748 
3749         for (; up_svcs[i] != NULL; ++i) {
3750                 if (up_svcs_p[i] == NULL) {
3751                         up_svcs_p[i] = vertex_get_by_name(up_svcs[i]);
3752 
3753                         if (up_svcs_p[i] == NULL)
3754                                 continue;
3755                 }
3756 
3757                 /*
3758                  * Ignore unconfigured services (the ones that have been
3759                  * mentioned in a dependency from other services, but do
3760                  * not exist in the repository).  Services which exist
3761                  * in the repository but don't have general/enabled
3762                  * property will be also ignored.
3763                  */
3764                 if (!(up_svcs_p[i]->gv_flags & GV_CONFIGURED))
3765                         continue;
3766 
3767                 switch (up_svcs_p[i]->gv_state) {
3768                 case RESTARTER_STATE_ONLINE:
3769                 case RESTARTER_STATE_DEGRADED:
3770                         /*
3771                          * Deactivate verbose boot once a login service has been
3772                          * reached.
3773                          */
3774                         st->st_log_login_reached = 1;
3775                         /*FALLTHROUGH*/
3776                 case RESTARTER_STATE_UNINIT:
3777                         return (B_TRUE);
3778 
3779                 case RESTARTER_STATE_OFFLINE:
3780                         if (instance_satisfied(up_svcs_p[i], B_TRUE) != -1)
3781                                 return (B_TRUE);
3782                         log_framework(LOG_DEBUG,
3783                             "can_come_up(): %s is unsatisfiable.\n",
3784                             up_svcs_p[i]->gv_name);
3785                         continue;
3786 
3787                 case RESTARTER_STATE_DISABLED:
3788                 case RESTARTER_STATE_MAINT:
3789                         log_framework(LOG_DEBUG,
3790                             "can_come_up(): %s is in state %s.\n",
3791                             up_svcs_p[i]->gv_name,
3792                             instance_state_str[up_svcs_p[i]->gv_state]);
3793                         continue;
3794 
3795                 default:
3796 #ifndef NDEBUG
3797                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
3798                             __FILE__, __LINE__, up_svcs_p[i]->gv_state);
3799 #endif
3800                         abort();
3801                 }
3802         }
3803 
3804         /*
3805          * In the seed repository, console-login is unsatisfiable because
3806          * services are missing.  To behave correctly in that case we don't want
3807          * to return false until manifest-import is online.
3808          */
3809 
3810         if (manifest_import_p == NULL) {
3811                 manifest_import_p = vertex_get_by_name(manifest_import);
3812 
3813                 if (manifest_import_p == NULL)
3814                         return (B_FALSE);
3815         }
3816 
3817         switch (manifest_import_p->gv_state) {
3818         case RESTARTER_STATE_ONLINE:
3819         case RESTARTER_STATE_DEGRADED:
3820         case RESTARTER_STATE_DISABLED:
3821         case RESTARTER_STATE_MAINT:
3822                 break;
3823 
3824         case RESTARTER_STATE_OFFLINE:
3825                 if (instance_satisfied(manifest_import_p, B_TRUE) == -1)
3826                         break;
3827                 /* FALLTHROUGH */
3828 
3829         case RESTARTER_STATE_UNINIT:
3830                 return (B_TRUE);
3831         }
3832 
3833         return (B_FALSE);
3834 }
3835 
3836 /*
3837  * Runs sulogin.  Returns
3838  *   0 - success
3839  *   EALREADY - sulogin is already running
3840  *   EBUSY - console-login is running
3841  */
3842 static int
3843 run_sulogin(const char *msg)
3844 {
3845         graph_vertex_t *v;
3846 
3847         assert(MUTEX_HELD(&dgraph_lock));
3848 
3849         if (sulogin_running)
3850                 return (EALREADY);
3851 
3852         v = vertex_get_by_name(console_login_fmri);
3853         if (v != NULL && inst_running(v))
3854                 return (EBUSY);
3855 
3856         sulogin_running = B_TRUE;
3857 
3858         MUTEX_UNLOCK(&dgraph_lock);
3859 
3860         fork_sulogin(B_FALSE, msg);
3861 
3862         MUTEX_LOCK(&dgraph_lock);
3863 
3864         sulogin_running = B_FALSE;
3865 
3866         if (console_login_ready) {
3867                 v = vertex_get_by_name(console_login_fmri);
3868 
3869                 if (v != NULL && v->gv_state == RESTARTER_STATE_OFFLINE) {
3870                         if (v->gv_start_f == NULL)
3871                                 vertex_send_event(v,
3872                                     RESTARTER_EVENT_TYPE_START);
3873                         else
3874                                 v->gv_start_f(v);
3875                 }
3876 
3877                 console_login_ready = B_FALSE;
3878         }
3879 
3880         return (0);
3881 }
3882 
3883 /*
3884  * The sulogin thread runs sulogin while can_come_up() is false.  run_sulogin()
3885  * keeps sulogin from stepping on console-login's toes.
3886  */
3887 /* ARGSUSED */
3888 static void *
3889 sulogin_thread(void *unused)
3890 {
3891         MUTEX_LOCK(&dgraph_lock);
3892 
3893         assert(sulogin_thread_running);
3894 
3895         do {
3896                 (void) run_sulogin("Console login service(s) cannot run\n");
3897         } while (!can_come_up());
3898 
3899         sulogin_thread_running = B_FALSE;
3900         MUTEX_UNLOCK(&dgraph_lock);
3901 
3902         return (NULL);
3903 }
3904 
3905 /* ARGSUSED */
3906 void *
3907 single_user_thread(void *unused)
3908 {
3909         uint_t left;
3910         scf_handle_t *h;
3911         scf_instance_t *inst;
3912         scf_property_t *prop;
3913         scf_value_t *val;
3914         const char *msg;
3915         char *buf;
3916         int r;
3917 
3918         MUTEX_LOCK(&single_user_thread_lock);
3919         single_user_thread_count++;
3920 
3921         if (!booting_to_single_user)
3922                 kill_user_procs();
3923 
3924         if (go_single_user_mode || booting_to_single_user) {
3925                 msg = "SINGLE USER MODE\n";
3926         } else {
3927                 assert(go_to_level1);
3928 
3929                 fork_rc_script('1', "start", B_TRUE);
3930 
3931                 uu_warn("The system is ready for administration.\n");
3932 
3933                 msg = "";
3934         }
3935 
3936         MUTEX_UNLOCK(&single_user_thread_lock);
3937 
3938         for (;;) {
3939                 MUTEX_LOCK(&dgraph_lock);
3940                 r = run_sulogin(msg);
3941                 MUTEX_UNLOCK(&dgraph_lock);
3942                 if (r == 0)
3943                         break;
3944 
3945                 assert(r == EALREADY || r == EBUSY);
3946 
3947                 left = 3;
3948                 while (left > 0)
3949                         left = sleep(left);
3950         }
3951 
3952         MUTEX_LOCK(&single_user_thread_lock);
3953 
3954         /*
3955          * If another single user thread has started, let it finish changing
3956          * the run level.
3957          */
3958         if (single_user_thread_count > 1) {
3959                 single_user_thread_count--;
3960                 MUTEX_UNLOCK(&single_user_thread_lock);
3961                 return (NULL);
3962         }
3963 
3964         h = libscf_handle_create_bound_loop();
3965         inst = scf_instance_create(h);
3966         prop = safe_scf_property_create(h);
3967         val = safe_scf_value_create(h);
3968         buf = startd_alloc(max_scf_fmri_size);
3969 
3970 lookup:
3971         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
3972             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
3973                 switch (scf_error()) {
3974                 case SCF_ERROR_NOT_FOUND:
3975                         r = libscf_create_self(h);
3976                         if (r == 0)
3977                                 goto lookup;
3978                         assert(r == ECONNABORTED);
3979                         /* FALLTHROUGH */
3980 
3981                 case SCF_ERROR_CONNECTION_BROKEN:
3982                         libscf_handle_rebind(h);
3983                         goto lookup;
3984 
3985                 case SCF_ERROR_INVALID_ARGUMENT:
3986                 case SCF_ERROR_CONSTRAINT_VIOLATED:
3987                 case SCF_ERROR_NOT_BOUND:
3988                 case SCF_ERROR_HANDLE_MISMATCH:
3989                 default:
3990                         bad_error("scf_handle_decode_fmri", scf_error());
3991                 }
3992         }
3993 
3994         MUTEX_LOCK(&dgraph_lock);
3995 
3996         r = scf_instance_delete_prop(inst, SCF_PG_OPTIONS_OVR,
3997             SCF_PROPERTY_MILESTONE);
3998         switch (r) {
3999         case 0:
4000         case ECANCELED:
4001                 break;
4002 
4003         case ECONNABORTED:
4004                 MUTEX_UNLOCK(&dgraph_lock);
4005                 libscf_handle_rebind(h);
4006                 goto lookup;
4007 
4008         case EPERM:
4009         case EACCES:
4010         case EROFS:
4011                 log_error(LOG_WARNING, "Could not clear temporary milestone: "
4012                     "%s.\n", strerror(r));
4013                 break;
4014 
4015         default:
4016                 bad_error("scf_instance_delete_prop", r);
4017         }
4018 
4019         MUTEX_UNLOCK(&dgraph_lock);
4020 
4021         r = libscf_get_milestone(inst, prop, val, buf, max_scf_fmri_size);
4022         switch (r) {
4023         case ECANCELED:
4024         case ENOENT:
4025         case EINVAL:
4026                 (void) strcpy(buf, "all");
4027                 /* FALLTHROUGH */
4028 
4029         case 0:
4030                 uu_warn("Returning to milestone %s.\n", buf);
4031                 break;
4032 
4033         case ECONNABORTED:
4034                 libscf_handle_rebind(h);
4035                 goto lookup;
4036 
4037         default:
4038                 bad_error("libscf_get_milestone", r);
4039         }
4040 
4041         r = dgraph_set_milestone(buf, h, B_FALSE);
4042         switch (r) {
4043         case 0:
4044         case ECONNRESET:
4045         case EALREADY:
4046         case EINVAL:
4047         case ENOENT:
4048                 break;
4049 
4050         default:
4051                 bad_error("dgraph_set_milestone", r);
4052         }
4053 
4054         /*
4055          * See graph_runlevel_changed().
4056          */
4057         MUTEX_LOCK(&dgraph_lock);
4058         utmpx_set_runlevel(target_milestone_as_runlevel(), 'S', B_TRUE);
4059         MUTEX_UNLOCK(&dgraph_lock);
4060 
4061         startd_free(buf, max_scf_fmri_size);
4062         scf_value_destroy(val);
4063         scf_property_destroy(prop);
4064         scf_instance_destroy(inst);
4065         scf_handle_destroy(h);
4066 
4067         /*
4068          * We'll give ourselves 3 seconds to respond to all of the enablings
4069          * that setting the milestone should have created before checking
4070          * whether to run sulogin.
4071          */
4072         left = 3;
4073         while (left > 0)
4074                 left = sleep(left);
4075 
4076         MUTEX_LOCK(&dgraph_lock);
4077         /*
4078          * Clearing these variables will allow the sulogin thread to run.  We
4079          * check here in case there aren't any more state updates anytime soon.
4080          */
4081         go_to_level1 = go_single_user_mode = booting_to_single_user = B_FALSE;
4082         if (!sulogin_thread_running && !can_come_up()) {
4083                 (void) startd_thread_create(sulogin_thread, NULL);
4084                 sulogin_thread_running = B_TRUE;
4085         }
4086         MUTEX_UNLOCK(&dgraph_lock);
4087         single_user_thread_count--;
4088         MUTEX_UNLOCK(&single_user_thread_lock);
4089         return (NULL);
4090 }
4091 
4092 
4093 /*
4094  * Dependency graph operations API.  These are handle-independent thread-safe
4095  * graph manipulation functions which are the entry points for the event
4096  * threads below.
4097  */
4098 
4099 /*
4100  * If a configured vertex exists for inst_fmri, return EEXIST.  If no vertex
4101  * exists for inst_fmri, add one.  Then fetch the restarter from inst, make
4102  * this vertex dependent on it, and send _ADD_INSTANCE to the restarter.
4103  * Fetch whether the instance should be enabled from inst and send _ENABLE or
4104  * _DISABLE as appropriate.  Finally rummage through inst's dependency
4105  * property groups and add vertices and edges as appropriate.  If anything
4106  * goes wrong after sending _ADD_INSTANCE, send _ADMIN_MAINT_ON to put the
4107  * instance in maintenance.  Don't send _START or _STOP until we get a state
4108  * update in case we're being restarted and the service is already running.
4109  *
4110  * To support booting to a milestone, we must also make sure all dependencies
4111  * encountered are configured, if they exist in the repository.
4112  *
4113  * Returns 0 on success, ECONNABORTED on repository disconnection, EINVAL if
4114  * inst_fmri is an invalid (or not canonical) FMRI, ECANCELED if inst is
4115  * deleted, or EEXIST if a configured vertex for inst_fmri already exists.
4116  */
4117 int
4118 dgraph_add_instance(const char *inst_fmri, scf_instance_t *inst,
4119     boolean_t lock_graph)
4120 {
4121         graph_vertex_t *v;
4122         int err;
4123 
4124         if (strcmp(inst_fmri, SCF_SERVICE_STARTD) == 0)
4125                 return (0);
4126 
4127         /* Check for a vertex for inst_fmri. */
4128         if (lock_graph) {
4129                 MUTEX_LOCK(&dgraph_lock);
4130         } else {
4131                 assert(MUTEX_HELD(&dgraph_lock));
4132         }
4133 
4134         v = vertex_get_by_name(inst_fmri);
4135 
4136         if (v != NULL) {
4137                 assert(v->gv_type == GVT_INST);
4138 
4139                 if (v->gv_flags & GV_CONFIGURED) {
4140                         if (lock_graph)
4141                                 MUTEX_UNLOCK(&dgraph_lock);
4142                         return (EEXIST);
4143                 }
4144         } else {
4145                 /* Add the vertex. */
4146                 err = graph_insert_vertex_unconfigured(inst_fmri, GVT_INST, 0,
4147                     RERR_NONE, &v);
4148                 if (err != 0) {
4149                         assert(err == EINVAL);
4150                         if (lock_graph)
4151                                 MUTEX_UNLOCK(&dgraph_lock);
4152                         return (EINVAL);
4153                 }
4154         }
4155 
4156         err = configure_vertex(v, inst);
4157 
4158         if (lock_graph)
4159                 MUTEX_UNLOCK(&dgraph_lock);
4160 
4161         return (err);
4162 }
4163 
4164 /*
4165  * Locate the vertex for this property group's instance.  If it doesn't exist
4166  * or is unconfigured, call dgraph_add_instance() & return.  Otherwise fetch
4167  * the restarter for the instance, and if it has changed, send
4168  * _REMOVE_INSTANCE to the old restarter, remove the dependency, make sure the
4169  * new restarter has a vertex, add a new dependency, and send _ADD_INSTANCE to
4170  * the new restarter.  Then fetch whether the instance should be enabled, and
4171  * if it is different from what we had, or if we changed the restarter, send
4172  * the appropriate _ENABLE or _DISABLE command.
4173  *
4174  * Returns 0 on success, ENOTSUP if the pg's parent is not an instance,
4175  * ECONNABORTED on repository disconnection, ECANCELED if the instance is
4176  * deleted, or -1 if the instance's general property group is deleted or if
4177  * its enabled property is misconfigured.
4178  */
4179 static int
4180 dgraph_update_general(scf_propertygroup_t *pg)
4181 {
4182         scf_handle_t *h;
4183         scf_instance_t *inst;
4184         char *fmri;
4185         char *restarter_fmri;
4186         graph_vertex_t *v;
4187         int err;
4188         int enabled, enabled_ovr;
4189         int oldflags;
4190 
4191         /* Find the vertex for this service */
4192         h = scf_pg_handle(pg);
4193 
4194         inst = safe_scf_instance_create(h);
4195 
4196         if (scf_pg_get_parent_instance(pg, inst) != 0) {
4197                 switch (scf_error()) {
4198                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4199                         return (ENOTSUP);
4200 
4201                 case SCF_ERROR_CONNECTION_BROKEN:
4202                 default:
4203                         return (ECONNABORTED);
4204 
4205                 case SCF_ERROR_DELETED:
4206                         return (0);
4207 
4208                 case SCF_ERROR_NOT_SET:
4209                         bad_error("scf_pg_get_parent_instance", scf_error());
4210                 }
4211         }
4212 
4213         err = libscf_instance_get_fmri(inst, &fmri);
4214         switch (err) {
4215         case 0:
4216                 break;
4217 
4218         case ECONNABORTED:
4219                 scf_instance_destroy(inst);
4220                 return (ECONNABORTED);
4221 
4222         case ECANCELED:
4223                 scf_instance_destroy(inst);
4224                 return (0);
4225 
4226         default:
4227                 bad_error("libscf_instance_get_fmri", err);
4228         }
4229 
4230         log_framework(LOG_DEBUG,
4231             "Graph engine: Reloading general properties for %s.\n", fmri);
4232 
4233         MUTEX_LOCK(&dgraph_lock);
4234 
4235         v = vertex_get_by_name(fmri);
4236         if (v == NULL || !(v->gv_flags & GV_CONFIGURED)) {
4237                 /* Will get the up-to-date properties. */
4238                 MUTEX_UNLOCK(&dgraph_lock);
4239                 err = dgraph_add_instance(fmri, inst, B_TRUE);
4240                 startd_free(fmri, max_scf_fmri_size);
4241                 scf_instance_destroy(inst);
4242                 return (err == ECANCELED ? 0 : err);
4243         }
4244 
4245         /* Read enabled & restarter from repository. */
4246         restarter_fmri = startd_alloc(max_scf_value_size);
4247         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
4248             &enabled_ovr, &restarter_fmri);
4249         if (err != 0 || enabled == -1) {
4250                 MUTEX_UNLOCK(&dgraph_lock);
4251                 scf_instance_destroy(inst);
4252                 startd_free(fmri, max_scf_fmri_size);
4253 
4254                 switch (err) {
4255                 case ENOENT:
4256                 case 0:
4257                         startd_free(restarter_fmri, max_scf_value_size);
4258                         return (-1);
4259 
4260                 case ECONNABORTED:
4261                 case ECANCELED:
4262                         startd_free(restarter_fmri, max_scf_value_size);
4263                         return (err);
4264 
4265                 default:
4266                         bad_error("libscf_get_basic_instance_data", err);
4267                 }
4268         }
4269 
4270         oldflags = v->gv_flags;
4271         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
4272             (enabled ? GV_ENBLD_NOOVR : 0);
4273 
4274         if (enabled_ovr != -1)
4275                 enabled = enabled_ovr;
4276 
4277         /*
4278          * If GV_ENBLD_NOOVR has changed, then we need to re-evaluate the
4279          * subgraph.
4280          */
4281         if (milestone > MILESTONE_NONE && v->gv_flags != oldflags)
4282                 (void) eval_subgraph(v, h);
4283 
4284         scf_instance_destroy(inst);
4285 
4286         /* Ignore restarter change for now. */
4287 
4288         startd_free(restarter_fmri, max_scf_value_size);
4289         startd_free(fmri, max_scf_fmri_size);
4290 
4291         /*
4292          * Always send _ENABLE or _DISABLE.  We could avoid this if the
4293          * restarter didn't change and the enabled value didn't change, but
4294          * that's not easy to check and improbable anyway, so we'll just do
4295          * this.
4296          */
4297         graph_enable_by_vertex(v, enabled, 1);
4298 
4299         MUTEX_UNLOCK(&dgraph_lock);
4300 
4301         return (0);
4302 }
4303 
4304 /*
4305  * Delete all of the property group dependencies of v, update inst's running
4306  * snapshot, and add the dependencies in the new snapshot.  If any of the new
4307  * dependencies would create a cycle, send _ADMIN_MAINT_ON.  Otherwise
4308  * reevaluate v's dependencies, send _START or _STOP as appropriate, and do
4309  * the same for v's dependents.
4310  *
4311  * Returns
4312  *   0 - success
4313  *   ECONNABORTED - repository connection broken
4314  *   ECANCELED - inst was deleted
4315  *   EINVAL - inst is invalid (e.g., missing general/enabled)
4316  *   -1 - libscf_snapshots_refresh() failed
4317  */
4318 static int
4319 dgraph_refresh_instance(graph_vertex_t *v, scf_instance_t *inst)
4320 {
4321         int r;
4322         int enabled;
4323         int32_t tset;
4324 
4325         assert(MUTEX_HELD(&dgraph_lock));
4326         assert(v->gv_type == GVT_INST);
4327 
4328         /* Only refresh services with valid general/enabled properties. */
4329         r = libscf_get_basic_instance_data(scf_instance_handle(inst), inst,
4330             v->gv_name, &enabled, NULL, NULL);
4331         switch (r) {
4332         case 0:
4333                 break;
4334 
4335         case ECONNABORTED:
4336         case ECANCELED:
4337                 return (r);
4338 
4339         case ENOENT:
4340                 log_framework(LOG_DEBUG,
4341                     "Ignoring %s because it has no general property group.\n",
4342                     v->gv_name);
4343                 return (EINVAL);
4344 
4345         default:
4346                 bad_error("libscf_get_basic_instance_data", r);
4347         }
4348 
4349         if ((tset = libscf_get_stn_tset(inst)) == -1) {
4350                 log_framework(LOG_WARNING,
4351                     "Failed to get notification parameters for %s: %s\n",
4352                     v->gv_name, scf_strerror(scf_error()));
4353                 tset = 0;
4354         }
4355         v->gv_stn_tset = tset;
4356         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
4357                 stn_global = tset;
4358 
4359         if (enabled == -1)
4360                 return (EINVAL);
4361 
4362         r = libscf_snapshots_refresh(inst, v->gv_name);
4363         if (r != 0) {
4364                 if (r != -1)
4365                         bad_error("libscf_snapshots_refresh", r);
4366 
4367                 /* error logged */
4368                 return (r);
4369         }
4370 
4371         r = refresh_vertex(v, inst);
4372         if (r != 0 && r != ECONNABORTED)
4373                 bad_error("refresh_vertex", r);
4374         return (r);
4375 }
4376 
4377 /*
4378  * Returns true only if none of this service's dependents are 'up' -- online
4379  * or degraded (offline is considered down in this situation). This function
4380  * is somehow similar to is_nonsubgraph_leaf() but works on subtrees.
4381  */
4382 static boolean_t
4383 insubtree_dependents_down(graph_vertex_t *v)
4384 {
4385         graph_vertex_t *vv;
4386         graph_edge_t *e;
4387 
4388         assert(MUTEX_HELD(&dgraph_lock));
4389 
4390         for (e = uu_list_first(v->gv_dependents); e != NULL;
4391             e = uu_list_next(v->gv_dependents, e)) {
4392                 vv = e->ge_vertex;
4393                 if (vv->gv_type == GVT_INST) {
4394                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4395                                 continue;
4396 
4397                         if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4398                                 continue;
4399 
4400                         if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4401                             (vv->gv_state == RESTARTER_STATE_DEGRADED))
4402                                 return (B_FALSE);
4403                 } else {
4404                         /*
4405                          * Skip all excluded and optional_all dependencies
4406                          * and decide whether to offline the service based
4407                          * on restart_on attribute.
4408                          */
4409                         if (is_depgrp_bypassed(vv))
4410                                 continue;
4411 
4412                         /*
4413                          * For dependency groups or service vertices, keep
4414                          * traversing to see if instances are running.
4415                          */
4416                         if (insubtree_dependents_down(vv) == B_FALSE)
4417                                 return (B_FALSE);
4418                 }
4419         }
4420 
4421         return (B_TRUE);
4422 }
4423 
4424 /*
4425  * Returns true only if none of this service's dependents are 'up' -- online,
4426  * degraded, or offline.
4427  */
4428 static int
4429 is_nonsubgraph_leaf(graph_vertex_t *v)
4430 {
4431         graph_vertex_t *vv;
4432         graph_edge_t *e;
4433 
4434         assert(MUTEX_HELD(&dgraph_lock));
4435 
4436         for (e = uu_list_first(v->gv_dependents);
4437             e != NULL;
4438             e = uu_list_next(v->gv_dependents, e)) {
4439 
4440                 vv = e->ge_vertex;
4441                 if (vv->gv_type == GVT_INST) {
4442                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4443                                 continue;
4444 
4445                         if (vv->gv_flags & GV_INSUBGRAPH)
4446                                 continue;
4447 
4448                         if (up_state(vv->gv_state))
4449                                 return (0);
4450                 } else {
4451                         /*
4452                          * For dependency group or service vertices, keep
4453                          * traversing to see if instances are running.
4454                          *
4455                          * We should skip exclude_all dependencies otherwise
4456                          * the vertex will never be considered as a leaf
4457                          * if the dependent is offline. The main reason for
4458                          * this is that disable_nonsubgraph_leaves() skips
4459                          * exclusion dependencies.
4460                          */
4461                         if (vv->gv_type == GVT_GROUP &&
4462                             vv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4463                                 continue;
4464 
4465                         if (!is_nonsubgraph_leaf(vv))
4466                                 return (0);
4467                 }
4468         }
4469 
4470         return (1);
4471 }
4472 
4473 /*
4474  * Disable v temporarily.  Attempt to do this by setting its enabled override
4475  * property in the repository.  If that fails, send a _DISABLE command.
4476  * Returns 0 on success and ECONNABORTED if the repository connection is
4477  * broken.
4478  */
4479 static int
4480 disable_service_temporarily(graph_vertex_t *v, scf_handle_t *h)
4481 {
4482         const char * const emsg = "Could not temporarily disable %s because "
4483             "%s.  Will stop service anyways.  Repository status for the "
4484             "service may be inaccurate.\n";
4485         const char * const emsg_cbroken =
4486             "the repository connection was broken";
4487 
4488         scf_instance_t *inst;
4489         int r;
4490 
4491         inst = scf_instance_create(h);
4492         if (inst == NULL) {
4493                 char buf[100];
4494 
4495                 (void) snprintf(buf, sizeof (buf),
4496                     "scf_instance_create() failed (%s)",
4497                     scf_strerror(scf_error()));
4498                 log_error(LOG_WARNING, emsg, v->gv_name, buf);
4499 
4500                 graph_enable_by_vertex(v, 0, 0);
4501                 return (0);
4502         }
4503 
4504         r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
4505             NULL, NULL, SCF_DECODE_FMRI_EXACT);
4506         if (r != 0) {
4507                 switch (scf_error()) {
4508                 case SCF_ERROR_CONNECTION_BROKEN:
4509                         log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4510                         graph_enable_by_vertex(v, 0, 0);
4511                         return (ECONNABORTED);
4512 
4513                 case SCF_ERROR_NOT_FOUND:
4514                         return (0);
4515 
4516                 case SCF_ERROR_HANDLE_MISMATCH:
4517                 case SCF_ERROR_INVALID_ARGUMENT:
4518                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4519                 case SCF_ERROR_NOT_BOUND:
4520                 default:
4521                         bad_error("scf_handle_decode_fmri",
4522                             scf_error());
4523                 }
4524         }
4525 
4526         r = libscf_set_enable_ovr(inst, 0);
4527         switch (r) {
4528         case 0:
4529                 scf_instance_destroy(inst);
4530                 return (0);
4531 
4532         case ECANCELED:
4533                 scf_instance_destroy(inst);
4534                 return (0);
4535 
4536         case ECONNABORTED:
4537                 log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4538                 graph_enable_by_vertex(v, 0, 0);
4539                 return (ECONNABORTED);
4540 
4541         case EPERM:
4542                 log_error(LOG_WARNING, emsg, v->gv_name,
4543                     "the repository denied permission");
4544                 graph_enable_by_vertex(v, 0, 0);
4545                 return (0);
4546 
4547         case EROFS:
4548                 log_error(LOG_WARNING, emsg, v->gv_name,
4549                     "the repository is read-only");
4550                 graph_enable_by_vertex(v, 0, 0);
4551                 return (0);
4552 
4553         default:
4554                 bad_error("libscf_set_enable_ovr", r);
4555                 /* NOTREACHED */
4556         }
4557 }
4558 
4559 /*
4560  * Of the transitive instance dependencies of v, offline those which are
4561  * in the subtree and which are leaves (i.e., have no dependents which are
4562  * "up").
4563  */
4564 void
4565 offline_subtree_leaves(graph_vertex_t *v, void *arg)
4566 {
4567         assert(MUTEX_HELD(&dgraph_lock));
4568 
4569         /* If v isn't an instance, recurse on its dependencies. */
4570         if (v->gv_type != GVT_INST) {
4571                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4572                 return;
4573         }
4574 
4575         /*
4576          * If v is not in the subtree, so should all of its dependencies,
4577          * so do nothing.
4578          */
4579         if ((v->gv_flags & GV_TOOFFLINE) == 0)
4580                 return;
4581 
4582         /* If v isn't a leaf because it's already down, recurse. */
4583         if (!up_state(v->gv_state)) {
4584                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4585                 return;
4586         }
4587 
4588         /* if v is a leaf, offline it or disable it if it's the last one */
4589         if (insubtree_dependents_down(v) == B_TRUE) {
4590                 if (v->gv_flags & GV_TODISABLE)
4591                         vertex_send_event(v,
4592                             RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
4593                 else
4594                         offline_vertex(v);
4595         }
4596 }
4597 
4598 void
4599 graph_offline_subtree_leaves(graph_vertex_t *v, void *h)
4600 {
4601         graph_walk_dependencies(v, offline_subtree_leaves, (void *)h);
4602 }
4603 
4604 
4605 /*
4606  * Of the transitive instance dependencies of v, disable those which are not
4607  * in the subgraph and which are leaves (i.e., have no dependents which are
4608  * "up").
4609  */
4610 static void
4611 disable_nonsubgraph_leaves(graph_vertex_t *v, void *arg)
4612 {
4613         assert(MUTEX_HELD(&dgraph_lock));
4614 
4615         /*
4616          * We must skip exclusion dependencies because they are allowed to
4617          * complete dependency cycles.  This is correct because A's exclusion
4618          * dependency on B doesn't bear on the order in which they should be
4619          * stopped.  Indeed, the exclusion dependency should guarantee that
4620          * they are never online at the same time.
4621          */
4622         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4623                 return;
4624 
4625         /* If v isn't an instance, recurse on its dependencies. */
4626         if (v->gv_type != GVT_INST)
4627                 goto recurse;
4628 
4629         if ((v->gv_flags & GV_CONFIGURED) == 0)
4630                 /*
4631                  * Unconfigured instances should have no dependencies, but in
4632                  * case they ever get them,
4633                  */
4634                 goto recurse;
4635 
4636         /*
4637          * If v is in the subgraph, so should all of its dependencies, so do
4638          * nothing.
4639          */
4640         if (v->gv_flags & GV_INSUBGRAPH)
4641                 return;
4642 
4643         /* If v isn't a leaf because it's already down, recurse. */
4644         if (!up_state(v->gv_state))
4645                 goto recurse;
4646 
4647         /* If v is disabled but not down yet, be patient. */
4648         if ((v->gv_flags & GV_ENABLED) == 0)
4649                 return;
4650 
4651         /* If v is a leaf, disable it. */
4652         if (is_nonsubgraph_leaf(v))
4653                 (void) disable_service_temporarily(v, (scf_handle_t *)arg);
4654 
4655         return;
4656 
4657 recurse:
4658         graph_walk_dependencies(v, disable_nonsubgraph_leaves, arg);
4659 }
4660 
4661 static int
4662 stn_restarter_state(restarter_instance_state_t rstate)
4663 {
4664         static const struct statemap {
4665                 restarter_instance_state_t restarter_state;
4666                 int scf_state;
4667         } map[] = {
4668                 { RESTARTER_STATE_UNINIT, SCF_STATE_UNINIT },
4669                 { RESTARTER_STATE_MAINT, SCF_STATE_MAINT },
4670                 { RESTARTER_STATE_OFFLINE, SCF_STATE_OFFLINE },
4671                 { RESTARTER_STATE_DISABLED, SCF_STATE_DISABLED },
4672                 { RESTARTER_STATE_ONLINE, SCF_STATE_ONLINE },
4673                 { RESTARTER_STATE_DEGRADED, SCF_STATE_DEGRADED }
4674         };
4675 
4676         int i;
4677 
4678         for (i = 0; i < sizeof (map) / sizeof (map[0]); i++) {
4679                 if (rstate == map[i].restarter_state)
4680                         return (map[i].scf_state);
4681         }
4682 
4683         return (-1);
4684 }
4685 
4686 /*
4687  * State transition counters
4688  * Not incremented atomically - indicative only
4689  */
4690 static uint64_t stev_ct_maint;
4691 static uint64_t stev_ct_hwerr;
4692 static uint64_t stev_ct_service;
4693 static uint64_t stev_ct_global;
4694 static uint64_t stev_ct_noprefs;
4695 static uint64_t stev_ct_from_uninit;
4696 static uint64_t stev_ct_bad_state;
4697 static uint64_t stev_ct_ovr_prefs;
4698 
4699 static void
4700 dgraph_state_transition_notify(graph_vertex_t *v,
4701     restarter_instance_state_t old_state, restarter_str_t reason)
4702 {
4703         restarter_instance_state_t new_state = v->gv_state;
4704         int stn_transition, maint;
4705         int from, to;
4706         nvlist_t *attr;
4707         fmev_pri_t pri = FMEV_LOPRI;
4708         int raise = 0;
4709 
4710         if ((from = stn_restarter_state(old_state)) == -1 ||
4711             (to = stn_restarter_state(new_state)) == -1) {
4712                 stev_ct_bad_state++;
4713                 return;
4714         }
4715 
4716         stn_transition = from << 16 | to;
4717 
4718         maint = (to == SCF_STATE_MAINT || from == SCF_STATE_MAINT);
4719 
4720         if (maint) {
4721                 /*
4722                  * All transitions to/from maintenance state must raise
4723                  * an event.
4724                  */
4725                 raise++;
4726                 pri = FMEV_HIPRI;
4727                 stev_ct_maint++;
4728         } else if (reason == restarter_str_ct_ev_hwerr) {
4729                 /*
4730                  * All transitions caused by hardware fault must raise
4731                  * an event
4732                  */
4733                 raise++;
4734                 pri = FMEV_HIPRI;
4735                 stev_ct_hwerr++;
4736         } else if (stn_transition & v->gv_stn_tset) {
4737                 /*
4738                  * Specifically enabled event.
4739                  */
4740                 raise++;
4741                 stev_ct_service++;
4742         } else if (from == SCF_STATE_UNINIT) {
4743                 /*
4744                  * Only raise these if specifically selected above.
4745                  */
4746                 stev_ct_from_uninit++;
4747         } else if (stn_transition & stn_global &&
4748             (IS_ENABLED(v) == 1 || to == SCF_STATE_DISABLED)) {
4749                 raise++;
4750                 stev_ct_global++;
4751         } else {
4752                 stev_ct_noprefs++;
4753         }
4754 
4755         if (info_events_all) {
4756                 stev_ct_ovr_prefs++;
4757                 raise++;
4758         }
4759         if (!raise)
4760                 return;
4761 
4762         if (nvlist_alloc(&attr, NV_UNIQUE_NAME, 0) != 0 ||
4763             nvlist_add_string(attr, "fmri", v->gv_name) != 0 ||
4764             nvlist_add_uint32(attr, "reason-version",
4765             restarter_str_version()) || nvlist_add_string(attr, "reason-short",
4766             restarter_get_str_short(reason)) != 0 ||
4767             nvlist_add_string(attr, "reason-long",
4768             restarter_get_str_long(reason)) != 0 ||
4769             nvlist_add_int32(attr, "transition", stn_transition) != 0) {
4770                 log_framework(LOG_WARNING,
4771                     "FMEV: %s could not create nvlist for transition "
4772                     "event: %s\n", v->gv_name, strerror(errno));
4773                 nvlist_free(attr);
4774                 return;
4775         }
4776 
4777         if (fmev_rspublish_nvl(FMEV_RULESET_SMF, "state-transition",
4778             instance_state_str[new_state], pri, attr) != FMEV_SUCCESS) {
4779                 log_framework(LOG_DEBUG,
4780                     "FMEV: %s failed to publish transition event: %s\n",
4781                     v->gv_name, fmev_strerror(fmev_errno));
4782                 nvlist_free(attr);
4783         }
4784 }
4785 
4786 /*
4787  * Find the vertex for inst_name.  If it doesn't exist, return ENOENT.
4788  * Otherwise set its state to state.  If the instance has entered a state
4789  * which requires automatic action, take it (Uninitialized: do
4790  * dgraph_refresh_instance() without the snapshot update.  Disabled: if the
4791  * instance should be enabled, send _ENABLE.  Offline: if the instance should
4792  * be disabled, send _DISABLE, and if its dependencies are satisfied, send
4793  * _START.  Online, Degraded: if the instance wasn't running, update its start
4794  * snapshot.  Maintenance: no action.)
4795  *
4796  * Also fails with ECONNABORTED, or EINVAL if state is invalid.
4797  */
4798 static int
4799 dgraph_set_instance_state(scf_handle_t *h, const char *inst_name,
4800     protocol_states_t *states)
4801 {
4802         graph_vertex_t *v;
4803         int err = 0;
4804         restarter_instance_state_t old_state;
4805         restarter_instance_state_t state = states->ps_state;
4806         restarter_error_t serr = states->ps_err;
4807 
4808         MUTEX_LOCK(&dgraph_lock);
4809 
4810         v = vertex_get_by_name(inst_name);
4811         if (v == NULL) {
4812                 MUTEX_UNLOCK(&dgraph_lock);
4813                 return (ENOENT);
4814         }
4815 
4816         assert(v->gv_type == GVT_INST);
4817 
4818         switch (state) {
4819         case RESTARTER_STATE_UNINIT:
4820         case RESTARTER_STATE_DISABLED:
4821         case RESTARTER_STATE_OFFLINE:
4822         case RESTARTER_STATE_ONLINE:
4823         case RESTARTER_STATE_DEGRADED:
4824         case RESTARTER_STATE_MAINT:
4825                 break;
4826 
4827         default:
4828                 MUTEX_UNLOCK(&dgraph_lock);
4829                 return (EINVAL);
4830         }
4831 
4832         log_framework(LOG_DEBUG, "Graph noting %s %s -> %s.\n", v->gv_name,
4833             instance_state_str[v->gv_state], instance_state_str[state]);
4834 
4835         old_state = v->gv_state;
4836         v->gv_state = state;
4837 
4838         v->gv_reason = states->ps_reason;
4839         err = gt_transition(h, v, serr, old_state);
4840         if (err == 0 && v->gv_state != old_state) {
4841                 dgraph_state_transition_notify(v, old_state, states->ps_reason);
4842         }
4843 
4844         MUTEX_UNLOCK(&dgraph_lock);
4845         return (err);
4846 }
4847 
4848 /*
4849  * Handle state changes during milestone shutdown.  See
4850  * dgraph_set_milestone().  If the repository connection is broken,
4851  * ECONNABORTED will be returned, though a _DISABLE command will be sent for
4852  * the vertex anyway.
4853  */
4854 int
4855 vertex_subgraph_dependencies_shutdown(scf_handle_t *h, graph_vertex_t *v,
4856     restarter_instance_state_t old_state)
4857 {
4858         int was_up, now_up;
4859         int ret = 0;
4860 
4861         assert(v->gv_type == GVT_INST);
4862 
4863         /* Don't care if we're not going to a milestone. */
4864         if (milestone == NULL)
4865                 return (0);
4866 
4867         /* Don't care if we already finished coming down. */
4868         if (non_subgraph_svcs == 0)
4869                 return (0);
4870 
4871         /* Don't care if the service is in the subgraph. */
4872         if (v->gv_flags & GV_INSUBGRAPH)
4873                 return (0);
4874 
4875         /*
4876          * Update non_subgraph_svcs.  It is the number of non-subgraph
4877          * services which are in online, degraded, or offline.
4878          */
4879 
4880         was_up = up_state(old_state);
4881         now_up = up_state(v->gv_state);
4882 
4883         if (!was_up && now_up) {
4884                 ++non_subgraph_svcs;
4885         } else if (was_up && !now_up) {
4886                 --non_subgraph_svcs;
4887 
4888                 if (non_subgraph_svcs == 0) {
4889                         if (halting != -1) {
4890                                 do_uadmin();
4891                         } else if (go_single_user_mode || go_to_level1) {
4892                                 (void) startd_thread_create(single_user_thread,
4893                                     NULL);
4894                         }
4895                         return (0);
4896                 }
4897         }
4898 
4899         /* If this service is a leaf, it should be disabled. */
4900         if ((v->gv_flags & GV_ENABLED) && is_nonsubgraph_leaf(v)) {
4901                 int r;
4902 
4903                 r = disable_service_temporarily(v, h);
4904                 switch (r) {
4905                 case 0:
4906                         break;
4907 
4908                 case ECONNABORTED:
4909                         ret = ECONNABORTED;
4910                         break;
4911 
4912                 default:
4913                         bad_error("disable_service_temporarily", r);
4914                 }
4915         }
4916 
4917         /*
4918          * If the service just came down, propagate the disable to the newly
4919          * exposed leaves.
4920          */
4921         if (was_up && !now_up)
4922                 graph_walk_dependencies(v, disable_nonsubgraph_leaves,
4923                     (void *)h);
4924 
4925         return (ret);
4926 }
4927 
4928 /*
4929  * Decide whether to start up an sulogin thread after a service is
4930  * finished changing state.  Only need to do the full can_come_up()
4931  * evaluation if an instance is changing state, we're not halfway through
4932  * loading the thread, and we aren't shutting down or going to the single
4933  * user milestone.
4934  */
4935 void
4936 graph_transition_sulogin(restarter_instance_state_t state,
4937     restarter_instance_state_t old_state)
4938 {
4939         assert(MUTEX_HELD(&dgraph_lock));
4940 
4941         if (state != old_state && st->st_load_complete &&
4942             !go_single_user_mode && !go_to_level1 &&
4943             halting == -1) {
4944                 if (!sulogin_thread_running && !can_come_up()) {
4945                         (void) startd_thread_create(sulogin_thread, NULL);
4946                         sulogin_thread_running = B_TRUE;
4947                 }
4948         }
4949 }
4950 
4951 /*
4952  * Propagate a start, stop event, or a satisfiability event.
4953  *
4954  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4955  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4956  * full dependent graph to check for newly satisfied nodes.  This is
4957  * necessary for cases when non-direct dependents may be effected but direct
4958  * dependents may not (e.g. for optional_all evaluations, see the
4959  * propagate_satbility() comments).
4960  *
4961  * PROPAGATE_SAT should be used whenever a non-running service moves into
4962  * a state which can satisfy optional dependencies, like disabled or
4963  * maintenance.
4964  */
4965 void
4966 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4967     restarter_error_t rerr)
4968 {
4969         if (type == PROPAGATE_STOP) {
4970                 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4971         } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4972                 graph_walk_dependents(v, propagate_start, NULL);
4973 
4974                 if (type == PROPAGATE_SAT)
4975                         propagate_satbility(v);
4976         } else {
4977 #ifndef NDEBUG
4978                 uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4979                     __LINE__, type);
4980 #endif
4981                 abort();
4982         }
4983 }
4984 
4985 /*
4986  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4987  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4988  * all property group dependencies, and the dependency on the restarter,
4989  * disposing of vertices as appropriate.  If other vertices depend on this
4990  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4991  * returns 0.
4992  */
4993 static int
4994 dgraph_remove_instance(const char *fmri, scf_handle_t *h)
4995 {
4996         graph_vertex_t *v;
4997         graph_edge_t *e;
4998         uu_list_t *old_deps;
4999         int err;
5000 
5001         log_framework(LOG_DEBUG, "Graph engine: Removing %s.\n", fmri);
5002 
5003         MUTEX_LOCK(&dgraph_lock);
5004 
5005         v = vertex_get_by_name(fmri);
5006         if (v == NULL) {
5007                 MUTEX_UNLOCK(&dgraph_lock);
5008                 return (0);
5009         }
5010 
5011         /* Send restarter delete event. */
5012         if (v->gv_flags & GV_CONFIGURED)
5013                 graph_unset_restarter(v);
5014 
5015         if (milestone > MILESTONE_NONE) {
5016                 /*
5017                  * Make a list of v's current dependencies so we can
5018                  * reevaluate their GV_INSUBGRAPH flags after the dependencies
5019                  * are removed.
5020                  */
5021                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
5022 
5023                 err = uu_list_walk(v->gv_dependencies,
5024                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
5025                 assert(err == 0);
5026         }
5027 
5028         delete_instance_dependencies(v, B_TRUE);
5029 
5030         /*
5031          * Deleting an instance can both satisfy and unsatisfy dependencies,
5032          * depending on their type.  First propagate the stop as a RERR_RESTART
5033          * event -- deletion isn't a fault, just a normal stop.  This gives
5034          * dependent services the chance to do a clean shutdown.  Then, mark
5035          * the service as unconfigured and propagate the start event for the
5036          * optional_all dependencies that might have become satisfied.
5037          */
5038         graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
5039 
5040         v->gv_flags &= ~GV_CONFIGURED;
5041         v->gv_flags &= ~GV_DEATHROW;
5042 
5043         graph_walk_dependents(v, propagate_start, NULL);
5044         propagate_satbility(v);
5045 
5046         /*
5047          * If there are no (non-service) dependents, the vertex can be
5048          * completely removed.
5049          */
5050         if (v != milestone && v->gv_refs == 0 &&
5051             uu_list_numnodes(v->gv_dependents) == 1)
5052                 remove_inst_vertex(v);
5053 
5054         if (milestone > MILESTONE_NONE) {
5055                 void *cookie = NULL;
5056 
5057                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5058                         v = e->ge_vertex;
5059 
5060                         if (vertex_unref(v) == VERTEX_INUSE)
5061                                 while (eval_subgraph(v, h) == ECONNABORTED)
5062                                         libscf_handle_rebind(h);
5063 
5064                         startd_free(e, sizeof (*e));
5065                 }
5066 
5067                 uu_list_destroy(old_deps);
5068         }
5069 
5070         MUTEX_UNLOCK(&dgraph_lock);
5071 
5072         return (0);
5073 }
5074 
5075 /*
5076  * Return the eventual (maybe current) milestone in the form of a
5077  * legacy runlevel.
5078  */
5079 static char
5080 target_milestone_as_runlevel()
5081 {
5082         assert(MUTEX_HELD(&dgraph_lock));
5083 
5084         if (milestone == NULL)
5085                 return ('3');
5086         else if (milestone == MILESTONE_NONE)
5087                 return ('0');
5088 
5089         if (strcmp(milestone->gv_name, multi_user_fmri) == 0)
5090                 return ('2');
5091         else if (strcmp(milestone->gv_name, single_user_fmri) == 0)
5092                 return ('S');
5093         else if (strcmp(milestone->gv_name, multi_user_svr_fmri) == 0)
5094                 return ('3');
5095 
5096 #ifndef NDEBUG
5097         (void) fprintf(stderr, "%s:%d: Unknown milestone name \"%s\".\n",
5098             __FILE__, __LINE__, milestone->gv_name);
5099 #endif
5100         abort();
5101         /* NOTREACHED */
5102 }
5103 
5104 static struct {
5105         char    rl;
5106         int     sig;
5107 } init_sigs[] = {
5108         { 'S', SIGBUS },
5109         { '0', SIGINT },
5110         { '1', SIGQUIT },
5111         { '2', SIGILL },
5112         { '3', SIGTRAP },
5113         { '4', SIGIOT },
5114         { '5', SIGEMT },
5115         { '6', SIGFPE },
5116         { 0, 0 }
5117 };
5118 
5119 static void
5120 signal_init(char rl)
5121 {
5122         pid_t init_pid;
5123         int i;
5124 
5125         assert(MUTEX_HELD(&dgraph_lock));
5126 
5127         if (zone_getattr(getzoneid(), ZONE_ATTR_INITPID, &init_pid,
5128             sizeof (init_pid)) != sizeof (init_pid)) {
5129                 log_error(LOG_NOTICE, "Could not get pid to signal init.\n");
5130                 return;
5131         }
5132 
5133         for (i = 0; init_sigs[i].rl != 0; ++i)
5134                 if (init_sigs[i].rl == rl)
5135                         break;
5136 
5137         if (init_sigs[i].rl != 0) {
5138                 if (kill(init_pid, init_sigs[i].sig) != 0) {
5139                         switch (errno) {
5140                         case EPERM:
5141                         case ESRCH:
5142                                 log_error(LOG_NOTICE, "Could not signal init: "
5143                                     "%s.\n", strerror(errno));
5144                                 break;
5145 
5146                         case EINVAL:
5147                         default:
5148                                 bad_error("kill", errno);
5149                         }
5150                 }
5151         }
5152 }
5153 
5154 /*
5155  * This is called when one of the major milestones changes state, or when
5156  * init is signalled and tells us it was told to change runlevel.  We wait
5157  * to reach the milestone because this allows /etc/inittab entries to retain
5158  * some boot ordering: historically, entries could place themselves before/after
5159  * the running of /sbin/rcX scripts but we can no longer make the
5160  * distinction because the /sbin/rcX scripts no longer exist as punctuation
5161  * marks in /etc/inittab.
5162  *
5163  * Also, we only trigger an update when we reach the eventual target
5164  * milestone: without this, an /etc/inittab entry marked only for
5165  * runlevel 2 would be executed for runlevel 3, which is not how
5166  * /etc/inittab entries work.
5167  *
5168  * If we're single user coming online, then we set utmpx to the target
5169  * runlevel so that legacy scripts can work as expected.
5170  */
5171 static void
5172 graph_runlevel_changed(char rl, int online)
5173 {
5174         char trl;
5175 
5176         assert(MUTEX_HELD(&dgraph_lock));
5177 
5178         trl = target_milestone_as_runlevel();
5179 
5180         if (online) {
5181                 if (rl == trl) {
5182                         current_runlevel = trl;
5183                         signal_init(trl);
5184                 } else if (rl == 'S') {
5185                         /*
5186                          * At boot, set the entry early for the benefit of the
5187                          * legacy init scripts.
5188                          */
5189                         utmpx_set_runlevel(trl, 'S', B_FALSE);
5190                 }
5191         } else {
5192                 if (rl == '3' && trl == '2') {
5193                         current_runlevel = trl;
5194                         signal_init(trl);
5195                 } else if (rl == '2' && trl == 'S') {
5196                         current_runlevel = trl;
5197                         signal_init(trl);
5198                 }
5199         }
5200 }
5201 
5202 /*
5203  * Move to a backwards-compatible runlevel by executing the appropriate
5204  * /etc/rc?.d/K* scripts and/or setting the milestone.
5205  *
5206  * Returns
5207  *   0 - success
5208  *   ECONNRESET - success, but handle was reset
5209  *   ECONNABORTED - repository connection broken
5210  *   ECANCELED - pg was deleted
5211  */
5212 static int
5213 dgraph_set_runlevel(scf_propertygroup_t *pg, scf_property_t *prop)
5214 {
5215         char rl;
5216         scf_handle_t *h;
5217         int r;
5218         const char *ms = NULL;  /* what to commit as options/milestone */
5219         boolean_t rebound = B_FALSE;
5220         int mark_rl = 0;
5221 
5222         const char * const stop = "stop";
5223 
5224         r = libscf_extract_runlevel(prop, &rl);
5225         switch (r) {
5226         case 0:
5227                 break;
5228 
5229         case ECONNABORTED:
5230         case ECANCELED:
5231                 return (r);
5232 
5233         case EINVAL:
5234         case ENOENT:
5235                 log_error(LOG_WARNING, "runlevel property is misconfigured; "
5236                     "ignoring.\n");
5237                 /* delete the bad property */
5238                 goto nolock_out;
5239 
5240         default:
5241                 bad_error("libscf_extract_runlevel", r);
5242         }
5243 
5244         switch (rl) {
5245         case 's':
5246                 rl = 'S';
5247                 /* FALLTHROUGH */
5248 
5249         case 'S':
5250         case '2':
5251         case '3':
5252                 /*
5253                  * These cases cause a milestone change, so
5254                  * graph_runlevel_changed() will eventually deal with
5255                  * signalling init.
5256                  */
5257                 break;
5258 
5259         case '0':
5260         case '1':
5261         case '4':
5262         case '5':
5263         case '6':
5264                 mark_rl = 1;
5265                 break;
5266 
5267         default:
5268                 log_framework(LOG_NOTICE, "Unknown runlevel '%c'.\n", rl);
5269                 ms = NULL;
5270                 goto nolock_out;
5271         }
5272 
5273         h = scf_pg_handle(pg);
5274 
5275         MUTEX_LOCK(&dgraph_lock);
5276 
5277         /*
5278          * Since this triggers no milestone changes, force it by hand.
5279          */
5280         if (current_runlevel == '4' && rl == '3')
5281                 mark_rl = 1;
5282 
5283         /*
5284          * 1. If we are here after an "init X":
5285          *
5286          * init X
5287          *      init/lscf_set_runlevel()
5288          *              process_pg_event()
5289          *              dgraph_set_runlevel()
5290          *
5291          * then we haven't passed through graph_runlevel_changed() yet,
5292          * therefore 'current_runlevel' has not changed for sure but 'rl' has.
5293          * In consequence, if 'rl' is lower than 'current_runlevel', we change
5294          * the system runlevel and execute the appropriate /etc/rc?.d/K* scripts
5295          * past this test.
5296          *
5297          * 2. On the other hand, if we are here after a "svcadm milestone":
5298          *
5299          * svcadm milestone X
5300          *      dgraph_set_milestone()
5301          *              handle_graph_update_event()
5302          *              dgraph_set_instance_state()
5303          *              graph_post_X_[online|offline]()
5304          *              graph_runlevel_changed()
5305          *              signal_init()
5306          *                      init/lscf_set_runlevel()
5307          *                              process_pg_event()
5308          *                              dgraph_set_runlevel()
5309          *
5310          * then we already passed through graph_runlevel_changed() (by the way
5311          * of dgraph_set_milestone()) and 'current_runlevel' may have changed
5312          * and already be equal to 'rl' so we are going to return immediately
5313          * from dgraph_set_runlevel() without changing the system runlevel and
5314          * without executing the /etc/rc?.d/K* scripts.
5315          */
5316         if (rl == current_runlevel) {
5317                 ms = NULL;
5318                 goto out;
5319         }
5320 
5321         log_framework(LOG_DEBUG, "Changing to runlevel '%c'.\n", rl);
5322 
5323         /*
5324          * Make sure stop rc scripts see the new settings via who -r.
5325          */
5326         utmpx_set_runlevel(rl, current_runlevel, B_TRUE);
5327 
5328         /*
5329          * Some run levels don't have a direct correspondence to any
5330          * milestones, so we have to signal init directly.
5331          */
5332         if (mark_rl) {
5333                 current_runlevel = rl;
5334                 signal_init(rl);
5335         }
5336 
5337         switch (rl) {
5338         case 'S':
5339                 uu_warn("The system is coming down for administration.  "
5340                     "Please wait.\n");
5341                 fork_rc_script(rl, stop, B_FALSE);
5342                 ms = single_user_fmri;
5343                 go_single_user_mode = B_TRUE;
5344                 break;
5345 
5346         case '0':
5347                 halting_time = time(NULL);
5348                 fork_rc_script(rl, stop, B_TRUE);
5349                 halting = AD_HALT;
5350                 goto uadmin;
5351 
5352         case '5':
5353                 halting_time = time(NULL);
5354                 fork_rc_script(rl, stop, B_TRUE);
5355                 halting = AD_POWEROFF;
5356                 goto uadmin;
5357 
5358         case '6':
5359                 halting_time = time(NULL);
5360                 fork_rc_script(rl, stop, B_TRUE);
5361                 if (scf_is_fastboot_default() && getzoneid() == GLOBAL_ZONEID)
5362                         halting = AD_FASTREBOOT;
5363                 else
5364                         halting = AD_BOOT;
5365 
5366 uadmin:
5367                 uu_warn("The system is coming down.  Please wait.\n");
5368                 ms = "none";
5369 
5370                 /*
5371                  * We can't wait until all services are offline since this
5372                  * thread is responsible for taking them offline.  Instead we
5373                  * set halting to the second argument for uadmin() and call
5374                  * do_uadmin() from dgraph_set_instance_state() when
5375                  * appropriate.
5376                  */
5377                 break;
5378 
5379         case '1':
5380                 if (current_runlevel != 'S') {
5381                         uu_warn("Changing to state 1.\n");
5382                         fork_rc_script(rl, stop, B_FALSE);
5383                 } else {
5384                         uu_warn("The system is coming up for administration.  "
5385                             "Please wait.\n");
5386                 }
5387                 ms = single_user_fmri;
5388                 go_to_level1 = B_TRUE;
5389                 break;
5390 
5391         case '2':
5392                 if (current_runlevel == '3' || current_runlevel == '4')
5393                         fork_rc_script(rl, stop, B_FALSE);
5394                 ms = multi_user_fmri;
5395                 break;
5396 
5397         case '3':
5398         case '4':
5399                 ms = "all";
5400                 break;
5401 
5402         default:
5403 #ifndef NDEBUG
5404                 (void) fprintf(stderr, "%s:%d: Uncaught case %d ('%c').\n",
5405                     __FILE__, __LINE__, rl, rl);
5406 #endif
5407                 abort();
5408         }
5409 
5410 out:
5411         MUTEX_UNLOCK(&dgraph_lock);
5412 
5413 nolock_out:
5414         switch (r = libscf_clear_runlevel(pg, ms)) {
5415         case 0:
5416                 break;
5417 
5418         case ECONNABORTED:
5419                 libscf_handle_rebind(h);
5420                 rebound = B_TRUE;
5421                 goto nolock_out;
5422 
5423         case ECANCELED:
5424                 break;
5425 
5426         case EPERM:
5427         case EACCES:
5428         case EROFS:
5429                 log_error(LOG_NOTICE, "Could not delete \"%s/%s\" property: "
5430                     "%s.\n", SCF_PG_OPTIONS, "runlevel", strerror(r));
5431                 break;
5432 
5433         default:
5434                 bad_error("libscf_clear_runlevel", r);
5435         }
5436 
5437         return (rebound ? ECONNRESET : 0);
5438 }
5439 
5440 /*
5441  * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5442  * to the instances that are supposed to go offline during an
5443  * administrative disable operation.
5444  */
5445 static int
5446 mark_subtree(graph_edge_t *e, void *arg)
5447 {
5448         graph_vertex_t *v;
5449         int r;
5450 
5451         v = e->ge_vertex;
5452 
5453         /* If it's already in the subgraph, skip. */
5454         if (v->gv_flags & GV_TOOFFLINE)
5455                 return (UU_WALK_NEXT);
5456 
5457         switch (v->gv_type) {
5458         case GVT_INST:
5459                 /* If the instance is already disabled, skip it. */
5460                 if (!(v->gv_flags & GV_ENABLED))
5461                         return (UU_WALK_NEXT);
5462 
5463                 v->gv_flags |= GV_TOOFFLINE;
5464                 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5465                 break;
5466         case GVT_GROUP:
5467                 /*
5468                  * Skip all excluded and optional_all dependencies and decide
5469                  * whether to offline the service based on restart_on attribute.
5470                  */
5471                 if (is_depgrp_bypassed(v))
5472                         return (UU_WALK_NEXT);
5473                 break;
5474         }
5475 
5476         r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5477             0);
5478         assert(r == 0);
5479         return (UU_WALK_NEXT);
5480 }
5481 
5482 static int
5483 mark_subgraph(graph_edge_t *e, void *arg)
5484 {
5485         graph_vertex_t *v;
5486         int r;
5487         int optional = (int)arg;
5488 
5489         v = e->ge_vertex;
5490 
5491         /* If it's already in the subgraph, skip. */
5492         if (v->gv_flags & GV_INSUBGRAPH)
5493                 return (UU_WALK_NEXT);
5494 
5495         /*
5496          * Keep track if walk has entered an optional dependency group
5497          */
5498         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_OPTIONAL_ALL) {
5499                 optional = 1;
5500         }
5501         /*
5502          * Quit if we are in an optional dependency group and the instance
5503          * is disabled
5504          */
5505         if (optional && (v->gv_type == GVT_INST) &&
5506             (!(v->gv_flags & GV_ENBLD_NOOVR)))
5507                 return (UU_WALK_NEXT);
5508 
5509         v->gv_flags |= GV_INSUBGRAPH;
5510 
5511         /* Skip all excluded dependencies. */
5512         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
5513                 return (UU_WALK_NEXT);
5514 
5515         r = uu_list_walk(v->gv_dependencies, (uu_walk_fn_t *)mark_subgraph,
5516             (void *)optional, 0);
5517         assert(r == 0);
5518         return (UU_WALK_NEXT);
5519 }
5520 
5521 /*
5522  * Bring down all services which are not dependencies of fmri.  The
5523  * dependencies of fmri (direct & indirect) will constitute the "subgraph",
5524  * and will have the GV_INSUBGRAPH flag set.  The rest must be brought down,
5525  * which means the state is "disabled", "maintenance", or "uninitialized".  We
5526  * could consider "offline" to be down, and refrain from sending start
5527  * commands for such services, but that's not strictly necessary, so we'll
5528  * decline to intrude on the state machine.  It would probably confuse users
5529  * anyway.
5530  *
5531  * The services should be brought down in reverse-dependency order, so we
5532  * can't do it all at once here.  We initiate by override-disabling the leaves
5533  * of the dependency tree -- those services which are up but have no
5534  * dependents which are up.  When they come down,
5535  * vertex_subgraph_dependencies_shutdown() will override-disable the newly
5536  * exposed leaves.  Perseverance will ensure completion.
5537  *
5538  * Sometimes we need to take action when the transition is complete, like
5539  * start sulogin or halt the system.  To tell when we're done, we initialize
5540  * non_subgraph_svcs here to be the number of services which need to come
5541  * down.  As each does, we decrement the counter.  When it hits zero, we take
5542  * the appropriate action.  See vertex_subgraph_dependencies_shutdown().
5543  *
5544  * In case we're coming up, we also remove any enable-overrides for the
5545  * services which are dependencies of fmri.
5546  *
5547  * If norepository is true, the function will not change the repository.
5548  *
5549  * The decision to change the system run level in accordance with the milestone
5550  * is taken in dgraph_set_runlevel().
5551  *
5552  * Returns
5553  *   0 - success
5554  *   ECONNRESET - success, but handle was rebound
5555  *   EINVAL - fmri is invalid (error is logged)
5556  *   EALREADY - the milestone is already set to fmri
5557  *   ENOENT - a configured vertex does not exist for fmri (an error is logged)
5558  */
5559 static int
5560 dgraph_set_milestone(const char *fmri, scf_handle_t *h, boolean_t norepository)
5561 {
5562         const char *cfmri, *fs;
5563         graph_vertex_t *nm, *v;
5564         int ret = 0, r;
5565         scf_instance_t *inst;
5566         boolean_t isall, isnone, rebound = B_FALSE;
5567 
5568         /* Validate fmri */
5569         isall = (strcmp(fmri, "all") == 0);
5570         isnone = (strcmp(fmri, "none") == 0);
5571 
5572         if (!isall && !isnone) {
5573                 if (fmri_canonify(fmri, (char **)&cfmri, B_FALSE) == EINVAL)
5574                         goto reject;
5575 
5576                 if (strcmp(cfmri, single_user_fmri) != 0 &&
5577                     strcmp(cfmri, multi_user_fmri) != 0 &&
5578                     strcmp(cfmri, multi_user_svr_fmri) != 0) {
5579                         startd_free((void *)cfmri, max_scf_fmri_size);
5580 reject:
5581                         log_framework(LOG_WARNING,
5582                             "Rejecting request for invalid milestone \"%s\".\n",
5583                             fmri);
5584                         return (EINVAL);
5585                 }
5586         }
5587 
5588         inst = safe_scf_instance_create(h);
5589 
5590         MUTEX_LOCK(&dgraph_lock);
5591 
5592         if (milestone == NULL) {
5593                 if (isall) {
5594                         log_framework(LOG_DEBUG,
5595                             "Milestone already set to all.\n");
5596                         ret = EALREADY;
5597                         goto out;
5598                 }
5599         } else if (milestone == MILESTONE_NONE) {
5600                 if (isnone) {
5601                         log_framework(LOG_DEBUG,
5602                             "Milestone already set to none.\n");
5603                         ret = EALREADY;
5604                         goto out;
5605                 }
5606         } else {
5607                 if (!isall && !isnone &&
5608                     strcmp(cfmri, milestone->gv_name) == 0) {
5609                         log_framework(LOG_DEBUG,
5610                             "Milestone already set to %s.\n", cfmri);
5611                         ret = EALREADY;
5612                         goto out;
5613                 }
5614         }
5615 
5616         if (!isall && !isnone) {
5617                 nm = vertex_get_by_name(cfmri);
5618                 if (nm == NULL || !(nm->gv_flags & GV_CONFIGURED)) {
5619                         log_framework(LOG_WARNING, "Cannot set milestone to %s "
5620                             "because no such service exists.\n", cfmri);
5621                         ret = ENOENT;
5622                         goto out;
5623                 }
5624         }
5625 
5626         log_framework(LOG_DEBUG, "Changing milestone to %s.\n", fmri);
5627 
5628         /*
5629          * Set milestone, removing the old one if this was the last reference.
5630          */
5631         if (milestone > MILESTONE_NONE)
5632                 (void) vertex_unref(milestone);
5633 
5634         if (isall)
5635                 milestone = NULL;
5636         else if (isnone)
5637                 milestone = MILESTONE_NONE;
5638         else {
5639                 milestone = nm;
5640                 /* milestone should count as a reference */
5641                 vertex_ref(milestone);
5642         }
5643 
5644         /* Clear all GV_INSUBGRAPH bits. */
5645         for (v = uu_list_first(dgraph); v != NULL; v = uu_list_next(dgraph, v))
5646                 v->gv_flags &= ~GV_INSUBGRAPH;
5647 
5648         if (!isall && !isnone) {
5649                 /* Set GV_INSUBGRAPH for milestone & descendents. */
5650                 milestone->gv_flags |= GV_INSUBGRAPH;
5651 
5652                 r = uu_list_walk(milestone->gv_dependencies,
5653                     (uu_walk_fn_t *)mark_subgraph, NULL, 0);
5654                 assert(r == 0);
5655         }
5656 
5657         /* Un-override services in the subgraph & override-disable the rest. */
5658         if (norepository)
5659                 goto out;
5660 
5661         non_subgraph_svcs = 0;
5662         for (v = uu_list_first(dgraph);
5663             v != NULL;
5664             v = uu_list_next(dgraph, v)) {
5665                 if (v->gv_type != GVT_INST ||
5666                     (v->gv_flags & GV_CONFIGURED) == 0)
5667                         continue;
5668 
5669 again:
5670                 r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
5671                     NULL, NULL, SCF_DECODE_FMRI_EXACT);
5672                 if (r != 0) {
5673                         switch (scf_error()) {
5674                         case SCF_ERROR_CONNECTION_BROKEN:
5675                         default:
5676                                 libscf_handle_rebind(h);
5677                                 rebound = B_TRUE;
5678                                 goto again;
5679 
5680                         case SCF_ERROR_NOT_FOUND:
5681                                 continue;
5682 
5683                         case SCF_ERROR_HANDLE_MISMATCH:
5684                         case SCF_ERROR_INVALID_ARGUMENT:
5685                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5686                         case SCF_ERROR_NOT_BOUND:
5687                                 bad_error("scf_handle_decode_fmri",
5688                                     scf_error());
5689                         }
5690                 }
5691 
5692                 if (isall || (v->gv_flags & GV_INSUBGRAPH)) {
5693                         r = libscf_delete_enable_ovr(inst);
5694                         fs = "libscf_delete_enable_ovr";
5695                 } else {
5696                         assert(isnone || (v->gv_flags & GV_INSUBGRAPH) == 0);
5697 
5698                         /*
5699                          * Services which are up need to come down before
5700                          * we're done, but we can only disable the leaves
5701                          * here.
5702                          */
5703 
5704                         if (up_state(v->gv_state))
5705                                 ++non_subgraph_svcs;
5706 
5707                         /* If it's already disabled, don't bother. */
5708                         if ((v->gv_flags & GV_ENABLED) == 0)
5709                                 continue;
5710 
5711                         if (!is_nonsubgraph_leaf(v))
5712                                 continue;
5713 
5714                         r = libscf_set_enable_ovr(inst, 0);
5715                         fs = "libscf_set_enable_ovr";
5716                 }
5717                 switch (r) {
5718                 case 0:
5719                 case ECANCELED:
5720                         break;
5721 
5722                 case ECONNABORTED:
5723                         libscf_handle_rebind(h);
5724                         rebound = B_TRUE;
5725                         goto again;
5726 
5727                 case EPERM:
5728                 case EROFS:
5729                         log_error(LOG_WARNING,
5730                             "Could not set %s/%s for %s: %s.\n",
5731                             SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
5732                             v->gv_name, strerror(r));
5733                         break;
5734 
5735                 default:
5736                         bad_error(fs, r);
5737                 }
5738         }
5739 
5740         if (halting != -1) {
5741                 if (non_subgraph_svcs > 1)
5742                         uu_warn("%d system services are now being stopped.\n",
5743                             non_subgraph_svcs);
5744                 else if (non_subgraph_svcs == 1)
5745                         uu_warn("One system service is now being stopped.\n");
5746                 else if (non_subgraph_svcs == 0)
5747                         do_uadmin();
5748         }
5749 
5750         ret = rebound ? ECONNRESET : 0;
5751 
5752 out:
5753         MUTEX_UNLOCK(&dgraph_lock);
5754         if (!isall && !isnone)
5755                 startd_free((void *)cfmri, max_scf_fmri_size);
5756         scf_instance_destroy(inst);
5757         return (ret);
5758 }
5759 
5760 
5761 /*
5762  * Returns 0, ECONNABORTED, or EINVAL.
5763  */
5764 static int
5765 handle_graph_update_event(scf_handle_t *h, graph_protocol_event_t *e)
5766 {
5767         int r;
5768 
5769         switch (e->gpe_type) {
5770         case GRAPH_UPDATE_RELOAD_GRAPH:
5771                 log_error(LOG_WARNING,
5772                     "graph_event: reload graph unimplemented\n");
5773                 break;
5774 
5775         case GRAPH_UPDATE_STATE_CHANGE: {
5776                 protocol_states_t *states = e->gpe_data;
5777 
5778                 switch (r = dgraph_set_instance_state(h, e->gpe_inst, states)) {
5779                 case 0:
5780                 case ENOENT:
5781                         break;
5782 
5783                 case ECONNABORTED:
5784                         return (ECONNABORTED);
5785 
5786                 case EINVAL:
5787                 default:
5788 #ifndef NDEBUG
5789                         (void) fprintf(stderr, "dgraph_set_instance_state() "
5790                             "failed with unexpected error %d at %s:%d.\n", r,
5791                             __FILE__, __LINE__);
5792 #endif
5793                         abort();
5794                 }
5795 
5796                 startd_free(states, sizeof (protocol_states_t));
5797                 break;
5798         }
5799 
5800         default:
5801                 log_error(LOG_WARNING,
5802                     "graph_event_loop received an unknown event: %d\n",
5803                     e->gpe_type);
5804                 break;
5805         }
5806 
5807         return (0);
5808 }
5809 
5810 /*
5811  * graph_event_thread()
5812  *    Wait for state changes from the restarters.
5813  */
5814 /*ARGSUSED*/
5815 void *
5816 graph_event_thread(void *unused)
5817 {
5818         scf_handle_t *h;
5819         int err;
5820 
5821         h = libscf_handle_create_bound_loop();
5822 
5823         /*CONSTCOND*/
5824         while (1) {
5825                 graph_protocol_event_t *e;
5826 
5827                 MUTEX_LOCK(&gu->gu_lock);
5828 
5829                 while (gu->gu_wakeup == 0)
5830                         (void) pthread_cond_wait(&gu->gu_cv, &gu->gu_lock);
5831 
5832                 gu->gu_wakeup = 0;
5833 
5834                 while ((e = graph_event_dequeue()) != NULL) {
5835                         MUTEX_LOCK(&e->gpe_lock);
5836                         MUTEX_UNLOCK(&gu->gu_lock);
5837 
5838                         while ((err = handle_graph_update_event(h, e)) ==
5839                             ECONNABORTED)
5840                                 libscf_handle_rebind(h);
5841 
5842                         if (err == 0)
5843                                 graph_event_release(e);
5844                         else
5845                                 graph_event_requeue(e);
5846 
5847                         MUTEX_LOCK(&gu->gu_lock);
5848                 }
5849 
5850                 MUTEX_UNLOCK(&gu->gu_lock);
5851         }
5852 
5853         /*
5854          * Unreachable for now -- there's currently no graceful cleanup
5855          * called on exit().
5856          */
5857         MUTEX_UNLOCK(&gu->gu_lock);
5858         scf_handle_destroy(h);
5859         return (NULL);
5860 }
5861 
5862 static void
5863 set_initial_milestone(scf_handle_t *h)
5864 {
5865         scf_instance_t *inst;
5866         char *fmri, *cfmri;
5867         size_t sz;
5868         int r;
5869 
5870         inst = safe_scf_instance_create(h);
5871         fmri = startd_alloc(max_scf_fmri_size);
5872 
5873         /*
5874          * If -m milestone= was specified, we want to set options_ovr/milestone
5875          * to it.  Otherwise we want to read what the milestone should be set
5876          * to.  Either way we need our inst.
5877          */
5878 get_self:
5879         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
5880             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5881                 switch (scf_error()) {
5882                 case SCF_ERROR_CONNECTION_BROKEN:
5883                         libscf_handle_rebind(h);
5884                         goto get_self;
5885 
5886                 case SCF_ERROR_NOT_FOUND:
5887                         if (st->st_subgraph != NULL &&
5888                             st->st_subgraph[0] != '\0') {
5889                                 sz = strlcpy(fmri, st->st_subgraph,
5890                                     max_scf_fmri_size);
5891                                 assert(sz < max_scf_fmri_size);
5892                         } else {
5893                                 fmri[0] = '\0';
5894                         }
5895                         break;
5896 
5897                 case SCF_ERROR_INVALID_ARGUMENT:
5898                 case SCF_ERROR_CONSTRAINT_VIOLATED:
5899                 case SCF_ERROR_HANDLE_MISMATCH:
5900                 default:
5901                         bad_error("scf_handle_decode_fmri", scf_error());
5902                 }
5903         } else {
5904                 if (st->st_subgraph != NULL && st->st_subgraph[0] != '\0') {
5905                         scf_propertygroup_t *pg;
5906 
5907                         pg = safe_scf_pg_create(h);
5908 
5909                         sz = strlcpy(fmri, st->st_subgraph, max_scf_fmri_size);
5910                         assert(sz < max_scf_fmri_size);
5911 
5912                         r = libscf_inst_get_or_add_pg(inst, SCF_PG_OPTIONS_OVR,
5913                             SCF_PG_OPTIONS_OVR_TYPE, SCF_PG_OPTIONS_OVR_FLAGS,
5914                             pg);
5915                         switch (r) {
5916                         case 0:
5917                                 break;
5918 
5919                         case ECONNABORTED:
5920                                 libscf_handle_rebind(h);
5921                                 goto get_self;
5922 
5923                         case EPERM:
5924                         case EACCES:
5925                         case EROFS:
5926                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5927                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5928                                     SCF_PROPERTY_MILESTONE, strerror(r));
5929                                 /* FALLTHROUGH */
5930 
5931                         case ECANCELED:
5932                                 sz = strlcpy(fmri, st->st_subgraph,
5933                                     max_scf_fmri_size);
5934                                 assert(sz < max_scf_fmri_size);
5935                                 break;
5936 
5937                         default:
5938                                 bad_error("libscf_inst_get_or_add_pg", r);
5939                         }
5940 
5941                         r = libscf_clear_runlevel(pg, fmri);
5942                         switch (r) {
5943                         case 0:
5944                                 break;
5945 
5946                         case ECONNABORTED:
5947                                 libscf_handle_rebind(h);
5948                                 goto get_self;
5949 
5950                         case EPERM:
5951                         case EACCES:
5952                         case EROFS:
5953                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5954                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5955                                     SCF_PROPERTY_MILESTONE, strerror(r));
5956                                 /* FALLTHROUGH */
5957 
5958                         case ECANCELED:
5959                                 sz = strlcpy(fmri, st->st_subgraph,
5960                                     max_scf_fmri_size);
5961                                 assert(sz < max_scf_fmri_size);
5962                                 break;
5963 
5964                         default:
5965                                 bad_error("libscf_clear_runlevel", r);
5966                         }
5967 
5968                         scf_pg_destroy(pg);
5969                 } else {
5970                         scf_property_t *prop;
5971                         scf_value_t *val;
5972 
5973                         prop = safe_scf_property_create(h);
5974                         val = safe_scf_value_create(h);
5975 
5976                         r = libscf_get_milestone(inst, prop, val, fmri,
5977                             max_scf_fmri_size);
5978                         switch (r) {
5979                         case 0:
5980                                 break;
5981 
5982                         case ECONNABORTED:
5983                                 libscf_handle_rebind(h);
5984                                 goto get_self;
5985 
5986                         case EINVAL:
5987                                 log_error(LOG_WARNING, "Milestone property is "
5988                                     "misconfigured.  Defaulting to \"all\".\n");
5989                                 /* FALLTHROUGH */
5990 
5991                         case ECANCELED:
5992                         case ENOENT:
5993                                 fmri[0] = '\0';
5994                                 break;
5995 
5996                         default:
5997                                 bad_error("libscf_get_milestone", r);
5998                         }
5999 
6000                         scf_value_destroy(val);
6001                         scf_property_destroy(prop);
6002                 }
6003         }
6004 
6005         if (fmri[0] == '\0' || strcmp(fmri, "all") == 0)
6006                 goto out;
6007 
6008         if (strcmp(fmri, "none") != 0) {
6009 retry:
6010                 if (scf_handle_decode_fmri(h, fmri, NULL, NULL, inst, NULL,
6011                     NULL, SCF_DECODE_FMRI_EXACT) != 0) {
6012                         switch (scf_error()) {
6013                         case SCF_ERROR_INVALID_ARGUMENT:
6014                                 log_error(LOG_WARNING,
6015                                     "Requested milestone \"%s\" is invalid.  "
6016                                     "Reverting to \"all\".\n", fmri);
6017                                 goto out;
6018 
6019                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6020                                 log_error(LOG_WARNING, "Requested milestone "
6021                                     "\"%s\" does not specify an instance.  "
6022                                     "Reverting to \"all\".\n", fmri);
6023                                 goto out;
6024 
6025                         case SCF_ERROR_CONNECTION_BROKEN:
6026                                 libscf_handle_rebind(h);
6027                                 goto retry;
6028 
6029                         case SCF_ERROR_NOT_FOUND:
6030                                 log_error(LOG_WARNING, "Requested milestone "
6031                                     "\"%s\" not in repository.  Reverting to "
6032                                     "\"all\".\n", fmri);
6033                                 goto out;
6034 
6035                         case SCF_ERROR_HANDLE_MISMATCH:
6036                         default:
6037                                 bad_error("scf_handle_decode_fmri",
6038                                     scf_error());
6039                         }
6040                 }
6041 
6042                 r = fmri_canonify(fmri, &cfmri, B_FALSE);
6043                 assert(r == 0);
6044 
6045                 r = dgraph_add_instance(cfmri, inst, B_TRUE);
6046                 startd_free(cfmri, max_scf_fmri_size);
6047                 switch (r) {
6048                 case 0:
6049                         break;
6050 
6051                 case ECONNABORTED:
6052                         goto retry;
6053 
6054                 case EINVAL:
6055                         log_error(LOG_WARNING,
6056                             "Requested milestone \"%s\" is invalid.  "
6057                             "Reverting to \"all\".\n", fmri);
6058                         goto out;
6059 
6060                 case ECANCELED:
6061                         log_error(LOG_WARNING,
6062                             "Requested milestone \"%s\" not "
6063                             "in repository.  Reverting to \"all\".\n",
6064                             fmri);
6065                         goto out;
6066 
6067                 case EEXIST:
6068                 default:
6069                         bad_error("dgraph_add_instance", r);
6070                 }
6071         }
6072 
6073         log_console(LOG_INFO, "Booting to milestone \"%s\".\n", fmri);
6074 
6075         r = dgraph_set_milestone(fmri, h, B_FALSE);
6076         switch (r) {
6077         case 0:
6078         case ECONNRESET:
6079         case EALREADY:
6080                 break;
6081 
6082         case EINVAL:
6083         case ENOENT:
6084         default:
6085                 bad_error("dgraph_set_milestone", r);
6086         }
6087 
6088 out:
6089         startd_free(fmri, max_scf_fmri_size);
6090         scf_instance_destroy(inst);
6091 }
6092 
6093 void
6094 set_restart_milestone(scf_handle_t *h)
6095 {
6096         scf_instance_t *inst;
6097         scf_property_t *prop;
6098         scf_value_t *val;
6099         char *fmri;
6100         int r;
6101 
6102         inst = safe_scf_instance_create(h);
6103 
6104 get_self:
6105         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL,
6106             inst, NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
6107                 switch (scf_error()) {
6108                 case SCF_ERROR_CONNECTION_BROKEN:
6109                         libscf_handle_rebind(h);
6110                         goto get_self;
6111 
6112                 case SCF_ERROR_NOT_FOUND:
6113                         break;
6114 
6115                 case SCF_ERROR_INVALID_ARGUMENT:
6116                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6117                 case SCF_ERROR_HANDLE_MISMATCH:
6118                 default:
6119                         bad_error("scf_handle_decode_fmri", scf_error());
6120                 }
6121 
6122                 scf_instance_destroy(inst);
6123                 return;
6124         }
6125 
6126         prop = safe_scf_property_create(h);
6127         val = safe_scf_value_create(h);
6128         fmri = startd_alloc(max_scf_fmri_size);
6129 
6130         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6131         switch (r) {
6132         case 0:
6133                 break;
6134 
6135         case ECONNABORTED:
6136                 libscf_handle_rebind(h);
6137                 goto get_self;
6138 
6139         case ECANCELED:
6140         case ENOENT:
6141         case EINVAL:
6142                 goto out;
6143 
6144         default:
6145                 bad_error("libscf_get_milestone", r);
6146         }
6147 
6148         r = dgraph_set_milestone(fmri, h, B_TRUE);
6149         switch (r) {
6150         case 0:
6151         case ECONNRESET:
6152         case EALREADY:
6153         case EINVAL:
6154         case ENOENT:
6155                 break;
6156 
6157         default:
6158                 bad_error("dgraph_set_milestone", r);
6159         }
6160 
6161 out:
6162         startd_free(fmri, max_scf_fmri_size);
6163         scf_value_destroy(val);
6164         scf_property_destroy(prop);
6165         scf_instance_destroy(inst);
6166 }
6167 
6168 /*
6169  * void *graph_thread(void *)
6170  *
6171  * Graph management thread.
6172  */
6173 /*ARGSUSED*/
6174 void *
6175 graph_thread(void *arg)
6176 {
6177         scf_handle_t *h;
6178         int err;
6179 
6180         h = libscf_handle_create_bound_loop();
6181 
6182         if (st->st_initial)
6183                 set_initial_milestone(h);
6184 
6185         MUTEX_LOCK(&dgraph_lock);
6186         initial_milestone_set = B_TRUE;
6187         err = pthread_cond_broadcast(&initial_milestone_cv);
6188         assert(err == 0);
6189         MUTEX_UNLOCK(&dgraph_lock);
6190 
6191         libscf_populate_graph(h);
6192 
6193         if (!st->st_initial)
6194                 set_restart_milestone(h);
6195 
6196         MUTEX_LOCK(&st->st_load_lock);
6197         st->st_load_complete = 1;
6198         (void) pthread_cond_broadcast(&st->st_load_cv);
6199         MUTEX_UNLOCK(&st->st_load_lock);
6200 
6201         MUTEX_LOCK(&dgraph_lock);
6202         /*
6203          * Now that we've set st_load_complete we need to check can_come_up()
6204          * since if we booted to a milestone, then there won't be any more
6205          * state updates.
6206          */
6207         if (!go_single_user_mode && !go_to_level1 &&
6208             halting == -1) {
6209                 if (!sulogin_thread_running && !can_come_up()) {
6210                         (void) startd_thread_create(sulogin_thread, NULL);
6211                         sulogin_thread_running = B_TRUE;
6212                 }
6213         }
6214         MUTEX_UNLOCK(&dgraph_lock);
6215 
6216         (void) pthread_mutex_lock(&gu->gu_freeze_lock);
6217 
6218         /*CONSTCOND*/
6219         while (1) {
6220                 (void) pthread_cond_wait(&gu->gu_freeze_cv,
6221                     &gu->gu_freeze_lock);
6222         }
6223 
6224         /*
6225          * Unreachable for now -- there's currently no graceful cleanup
6226          * called on exit().
6227          */
6228         (void) pthread_mutex_unlock(&gu->gu_freeze_lock);
6229         scf_handle_destroy(h);
6230 
6231         return (NULL);
6232 }
6233 
6234 
6235 /*
6236  * int next_action()
6237  *   Given an array of timestamps 'a' with 'num' elements, find the
6238  *   lowest non-zero timestamp and return its index. If there are no
6239  *   non-zero elements, return -1.
6240  */
6241 static int
6242 next_action(hrtime_t *a, int num)
6243 {
6244         hrtime_t t = 0;
6245         int i = 0, smallest = -1;
6246 
6247         for (i = 0; i < num; i++) {
6248                 if (t == 0) {
6249                         t = a[i];
6250                         smallest = i;
6251                 } else if (a[i] != 0 && a[i] < t) {
6252                         t = a[i];
6253                         smallest = i;
6254                 }
6255         }
6256 
6257         if (t == 0)
6258                 return (-1);
6259         else
6260                 return (smallest);
6261 }
6262 
6263 /*
6264  * void process_actions()
6265  *   Process actions requested by the administrator. Possibilities include:
6266  *   refresh, restart, maintenance mode off, maintenance mode on,
6267  *   maintenance mode immediate, and degraded.
6268  *
6269  *   The set of pending actions is represented in the repository as a
6270  *   per-instance property group, with each action being a single property
6271  *   in that group.  This property group is converted to an array, with each
6272  *   action type having an array slot.  The actions in the array at the
6273  *   time process_actions() is called are acted on in the order of the
6274  *   timestamp (which is the value stored in the slot).  A value of zero
6275  *   indicates that there is no pending action of the type associated with
6276  *   a particular slot.
6277  *
6278  *   Sending an action event multiple times before the restarter has a
6279  *   chance to process that action will force it to be run at the last
6280  *   timestamp where it appears in the ordering.
6281  *
6282  *   Turning maintenance mode on trumps all other actions.
6283  *
6284  *   Returns 0 or ECONNABORTED.
6285  */
6286 static int
6287 process_actions(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst)
6288 {
6289         scf_property_t *prop = NULL;
6290         scf_value_t *val = NULL;
6291         scf_type_t type;
6292         graph_vertex_t *vertex;
6293         admin_action_t a;
6294         int i, ret = 0, r;
6295         hrtime_t action_ts[NACTIONS];
6296         char *inst_name;
6297 
6298         r = libscf_instance_get_fmri(inst, &inst_name);
6299         switch (r) {
6300         case 0:
6301                 break;
6302 
6303         case ECONNABORTED:
6304                 return (ECONNABORTED);
6305 
6306         case ECANCELED:
6307                 return (0);
6308 
6309         default:
6310                 bad_error("libscf_instance_get_fmri", r);
6311         }
6312 
6313         MUTEX_LOCK(&dgraph_lock);
6314 
6315         vertex = vertex_get_by_name(inst_name);
6316         if (vertex == NULL) {
6317                 MUTEX_UNLOCK(&dgraph_lock);
6318                 log_framework(LOG_DEBUG, "%s: Can't find graph vertex. "
6319                     "The instance must have been removed.\n", inst_name);
6320                 startd_free(inst_name, max_scf_fmri_size);
6321                 return (0);
6322         }
6323 
6324         prop = safe_scf_property_create(h);
6325         val = safe_scf_value_create(h);
6326 
6327         for (i = 0; i < NACTIONS; i++) {
6328                 if (scf_pg_get_property(pg, admin_actions[i], prop) != 0) {
6329                         switch (scf_error()) {
6330                         case SCF_ERROR_CONNECTION_BROKEN:
6331                         default:
6332                                 ret = ECONNABORTED;
6333                                 goto out;
6334 
6335                         case SCF_ERROR_DELETED:
6336                                 goto out;
6337 
6338                         case SCF_ERROR_NOT_FOUND:
6339                                 action_ts[i] = 0;
6340                                 continue;
6341 
6342                         case SCF_ERROR_HANDLE_MISMATCH:
6343                         case SCF_ERROR_INVALID_ARGUMENT:
6344                         case SCF_ERROR_NOT_SET:
6345                                 bad_error("scf_pg_get_property", scf_error());
6346                         }
6347                 }
6348 
6349                 if (scf_property_type(prop, &type) != 0) {
6350                         switch (scf_error()) {
6351                         case SCF_ERROR_CONNECTION_BROKEN:
6352                         default:
6353                                 ret = ECONNABORTED;
6354                                 goto out;
6355 
6356                         case SCF_ERROR_DELETED:
6357                                 action_ts[i] = 0;
6358                                 continue;
6359 
6360                         case SCF_ERROR_NOT_SET:
6361                                 bad_error("scf_property_type", scf_error());
6362                         }
6363                 }
6364 
6365                 if (type != SCF_TYPE_INTEGER) {
6366                         action_ts[i] = 0;
6367                         continue;
6368                 }
6369 
6370                 if (scf_property_get_value(prop, val) != 0) {
6371                         switch (scf_error()) {
6372                         case SCF_ERROR_CONNECTION_BROKEN:
6373                         default:
6374                                 ret = ECONNABORTED;
6375                                 goto out;
6376 
6377                         case SCF_ERROR_DELETED:
6378                                 goto out;
6379 
6380                         case SCF_ERROR_NOT_FOUND:
6381                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6382                                 action_ts[i] = 0;
6383                                 continue;
6384 
6385                         case SCF_ERROR_NOT_SET:
6386                         case SCF_ERROR_PERMISSION_DENIED:
6387                                 bad_error("scf_property_get_value",
6388                                     scf_error());
6389                         }
6390                 }
6391 
6392                 r = scf_value_get_integer(val, &action_ts[i]);
6393                 assert(r == 0);
6394         }
6395 
6396         a = ADMIN_EVENT_MAINT_ON_IMMEDIATE;
6397         if (action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ||
6398             action_ts[ADMIN_EVENT_MAINT_ON]) {
6399                 a = action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ?
6400                     ADMIN_EVENT_MAINT_ON_IMMEDIATE : ADMIN_EVENT_MAINT_ON;
6401 
6402                 vertex_send_event(vertex, admin_events[a]);
6403                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6404                 switch (r) {
6405                 case 0:
6406                 case EACCES:
6407                         break;
6408 
6409                 case ECONNABORTED:
6410                         ret = ECONNABORTED;
6411                         goto out;
6412 
6413                 case EPERM:
6414                         uu_die("Insufficient privilege.\n");
6415                         /* NOTREACHED */
6416 
6417                 default:
6418                         bad_error("libscf_unset_action", r);
6419                 }
6420         }
6421 
6422         while ((a = next_action(action_ts, NACTIONS)) != -1) {
6423                 log_framework(LOG_DEBUG,
6424                     "Graph: processing %s action for %s.\n", admin_actions[a],
6425                     inst_name);
6426 
6427                 if (a == ADMIN_EVENT_REFRESH) {
6428                         r = dgraph_refresh_instance(vertex, inst);
6429                         switch (r) {
6430                         case 0:
6431                         case ECANCELED:
6432                         case EINVAL:
6433                         case -1:
6434                                 break;
6435 
6436                         case ECONNABORTED:
6437                                 /* pg & inst are reset now, so just return. */
6438                                 ret = ECONNABORTED;
6439                                 goto out;
6440 
6441                         default:
6442                                 bad_error("dgraph_refresh_instance", r);
6443                         }
6444                 }
6445 
6446                 vertex_send_event(vertex, admin_events[a]);
6447 
6448                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6449                 switch (r) {
6450                 case 0:
6451                 case EACCES:
6452                         break;
6453 
6454                 case ECONNABORTED:
6455                         ret = ECONNABORTED;
6456                         goto out;
6457 
6458                 case EPERM:
6459                         uu_die("Insufficient privilege.\n");
6460                         /* NOTREACHED */
6461 
6462                 default:
6463                         bad_error("libscf_unset_action", r);
6464                 }
6465 
6466                 action_ts[a] = 0;
6467         }
6468 
6469 out:
6470         MUTEX_UNLOCK(&dgraph_lock);
6471 
6472         scf_property_destroy(prop);
6473         scf_value_destroy(val);
6474         startd_free(inst_name, max_scf_fmri_size);
6475         return (ret);
6476 }
6477 
6478 /*
6479  * inst and pg_name are scratch space, and are unset on entry.
6480  * Returns
6481  *   0 - success
6482  *   ECONNRESET - success, but repository handle rebound
6483  *   ECONNABORTED - repository connection broken
6484  */
6485 static int
6486 process_pg_event(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst,
6487     char *pg_name)
6488 {
6489         int r;
6490         scf_property_t *prop;
6491         scf_value_t *val;
6492         char *fmri;
6493         boolean_t rebound = B_FALSE, rebind_inst = B_FALSE;
6494 
6495         if (scf_pg_get_name(pg, pg_name, max_scf_value_size) < 0) {
6496                 switch (scf_error()) {
6497                 case SCF_ERROR_CONNECTION_BROKEN:
6498                 default:
6499                         return (ECONNABORTED);
6500 
6501                 case SCF_ERROR_DELETED:
6502                         return (0);
6503 
6504                 case SCF_ERROR_NOT_SET:
6505                         bad_error("scf_pg_get_name", scf_error());
6506                 }
6507         }
6508 
6509         if (strcmp(pg_name, SCF_PG_GENERAL) == 0 ||
6510             strcmp(pg_name, SCF_PG_GENERAL_OVR) == 0) {
6511                 r = dgraph_update_general(pg);
6512                 switch (r) {
6513                 case 0:
6514                 case ENOTSUP:
6515                 case ECANCELED:
6516                         return (0);
6517 
6518                 case ECONNABORTED:
6519                         return (ECONNABORTED);
6520 
6521                 case -1:
6522                         /* Error should have been logged. */
6523                         return (0);
6524 
6525                 default:
6526                         bad_error("dgraph_update_general", r);
6527                 }
6528         } else if (strcmp(pg_name, SCF_PG_RESTARTER_ACTIONS) == 0) {
6529                 if (scf_pg_get_parent_instance(pg, inst) != 0) {
6530                         switch (scf_error()) {
6531                         case SCF_ERROR_CONNECTION_BROKEN:
6532                                 return (ECONNABORTED);
6533 
6534                         case SCF_ERROR_DELETED:
6535                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6536                                 /* Ignore commands on services. */
6537                                 return (0);
6538 
6539                         case SCF_ERROR_NOT_BOUND:
6540                         case SCF_ERROR_HANDLE_MISMATCH:
6541                         case SCF_ERROR_NOT_SET:
6542                         default:
6543                                 bad_error("scf_pg_get_parent_instance",
6544                                     scf_error());
6545                         }
6546                 }
6547 
6548                 return (process_actions(h, pg, inst));
6549         }
6550 
6551         if (strcmp(pg_name, SCF_PG_OPTIONS) != 0 &&
6552             strcmp(pg_name, SCF_PG_OPTIONS_OVR) != 0)
6553                 return (0);
6554 
6555         /*
6556          * We only care about the options[_ovr] property groups of our own
6557          * instance, so get the fmri and compare.  Plus, once we know it's
6558          * correct, if the repository connection is broken we know exactly what
6559          * property group we were operating on, and can look it up again.
6560          */
6561         if (scf_pg_get_parent_instance(pg, inst) != 0) {
6562                 switch (scf_error()) {
6563                 case SCF_ERROR_CONNECTION_BROKEN:
6564                         return (ECONNABORTED);
6565 
6566                 case SCF_ERROR_DELETED:
6567                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6568                         return (0);
6569 
6570                 case SCF_ERROR_HANDLE_MISMATCH:
6571                 case SCF_ERROR_NOT_BOUND:
6572                 case SCF_ERROR_NOT_SET:
6573                 default:
6574                         bad_error("scf_pg_get_parent_instance",
6575                             scf_error());
6576                 }
6577         }
6578 
6579         switch (r = libscf_instance_get_fmri(inst, &fmri)) {
6580         case 0:
6581                 break;
6582 
6583         case ECONNABORTED:
6584                 return (ECONNABORTED);
6585 
6586         case ECANCELED:
6587                 return (0);
6588 
6589         default:
6590                 bad_error("libscf_instance_get_fmri", r);
6591         }
6592 
6593         if (strcmp(fmri, SCF_SERVICE_STARTD) != 0) {
6594                 startd_free(fmri, max_scf_fmri_size);
6595                 return (0);
6596         }
6597 
6598         /*
6599          * update the information events flag
6600          */
6601         if (strcmp(pg_name, SCF_PG_OPTIONS) == 0)
6602                 info_events_all = libscf_get_info_events_all(pg);
6603 
6604         prop = safe_scf_property_create(h);
6605         val = safe_scf_value_create(h);
6606 
6607         if (strcmp(pg_name, SCF_PG_OPTIONS_OVR) == 0) {
6608                 /* See if we need to set the runlevel. */
6609                 /* CONSTCOND */
6610                 if (0) {
6611 rebind_pg:
6612                         libscf_handle_rebind(h);
6613                         rebound = B_TRUE;
6614 
6615                         r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6616                         switch (r) {
6617                         case 0:
6618                                 break;
6619 
6620                         case ECONNABORTED:
6621                                 goto rebind_pg;
6622 
6623                         case ENOENT:
6624                                 goto out;
6625 
6626                         case EINVAL:
6627                         case ENOTSUP:
6628                                 bad_error("libscf_lookup_instance", r);
6629                         }
6630 
6631                         if (scf_instance_get_pg(inst, pg_name, pg) != 0) {
6632                                 switch (scf_error()) {
6633                                 case SCF_ERROR_DELETED:
6634                                 case SCF_ERROR_NOT_FOUND:
6635                                         goto out;
6636 
6637                                 case SCF_ERROR_CONNECTION_BROKEN:
6638                                         goto rebind_pg;
6639 
6640                                 case SCF_ERROR_HANDLE_MISMATCH:
6641                                 case SCF_ERROR_NOT_BOUND:
6642                                 case SCF_ERROR_NOT_SET:
6643                                 case SCF_ERROR_INVALID_ARGUMENT:
6644                                 default:
6645                                         bad_error("scf_instance_get_pg",
6646                                             scf_error());
6647                                 }
6648                         }
6649                 }
6650 
6651                 if (scf_pg_get_property(pg, "runlevel", prop) == 0) {
6652                         r = dgraph_set_runlevel(pg, prop);
6653                         switch (r) {
6654                         case ECONNRESET:
6655                                 rebound = B_TRUE;
6656                                 rebind_inst = B_TRUE;
6657                                 /* FALLTHROUGH */
6658 
6659                         case 0:
6660                                 break;
6661 
6662                         case ECONNABORTED:
6663                                 goto rebind_pg;
6664 
6665                         case ECANCELED:
6666                                 goto out;
6667 
6668                         default:
6669                                 bad_error("dgraph_set_runlevel", r);
6670                         }
6671                 } else {
6672                         switch (scf_error()) {
6673                         case SCF_ERROR_CONNECTION_BROKEN:
6674                         default:
6675                                 goto rebind_pg;
6676 
6677                         case SCF_ERROR_DELETED:
6678                                 goto out;
6679 
6680                         case SCF_ERROR_NOT_FOUND:
6681                                 break;
6682 
6683                         case SCF_ERROR_INVALID_ARGUMENT:
6684                         case SCF_ERROR_HANDLE_MISMATCH:
6685                         case SCF_ERROR_NOT_BOUND:
6686                         case SCF_ERROR_NOT_SET:
6687                                 bad_error("scf_pg_get_property", scf_error());
6688                         }
6689                 }
6690         }
6691 
6692         if (rebind_inst) {
6693 lookup_inst:
6694                 r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6695                 switch (r) {
6696                 case 0:
6697                         break;
6698 
6699                 case ECONNABORTED:
6700                         libscf_handle_rebind(h);
6701                         rebound = B_TRUE;
6702                         goto lookup_inst;
6703 
6704                 case ENOENT:
6705                         goto out;
6706 
6707                 case EINVAL:
6708                 case ENOTSUP:
6709                         bad_error("libscf_lookup_instance", r);
6710                 }
6711         }
6712 
6713         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6714         switch (r) {
6715         case 0:
6716                 break;
6717 
6718         case ECONNABORTED:
6719                 libscf_handle_rebind(h);
6720                 rebound = B_TRUE;
6721                 goto lookup_inst;
6722 
6723         case EINVAL:
6724                 log_error(LOG_NOTICE,
6725                     "%s/%s property of %s is misconfigured.\n", pg_name,
6726                     SCF_PROPERTY_MILESTONE, SCF_SERVICE_STARTD);
6727                 /* FALLTHROUGH */
6728 
6729         case ECANCELED:
6730         case ENOENT:
6731                 (void) strcpy(fmri, "all");
6732                 break;
6733 
6734         default:
6735                 bad_error("libscf_get_milestone", r);
6736         }
6737 
6738         r = dgraph_set_milestone(fmri, h, B_FALSE);
6739         switch (r) {
6740         case 0:
6741         case ECONNRESET:
6742         case EALREADY:
6743                 break;
6744 
6745         case EINVAL:
6746                 log_error(LOG_WARNING, "Milestone %s is invalid.\n", fmri);
6747                 break;
6748 
6749         case ENOENT:
6750                 log_error(LOG_WARNING, "Milestone %s does not exist.\n", fmri);
6751                 break;
6752 
6753         default:
6754                 bad_error("dgraph_set_milestone", r);
6755         }
6756 
6757 out:
6758         startd_free(fmri, max_scf_fmri_size);
6759         scf_value_destroy(val);
6760         scf_property_destroy(prop);
6761 
6762         return (rebound ? ECONNRESET : 0);
6763 }
6764 
6765 /*
6766  * process_delete() deletes an instance from the dgraph if 'fmri' is an
6767  * instance fmri or if 'fmri' matches the 'general' property group of an
6768  * instance (or the 'general/enabled' property).
6769  *
6770  * 'fmri' may be overwritten and cannot be trusted on return by the caller.
6771  */
6772 static void
6773 process_delete(char *fmri, scf_handle_t *h)
6774 {
6775         char *lfmri, *end_inst_fmri;
6776         const char *inst_name = NULL;
6777         const char *pg_name = NULL;
6778         const char *prop_name = NULL;
6779 
6780         lfmri = safe_strdup(fmri);
6781 
6782         /* Determine if the FMRI is a property group or instance */
6783         if (scf_parse_svc_fmri(lfmri, NULL, NULL, &inst_name, &pg_name,
6784             &prop_name) != SCF_SUCCESS) {
6785                 log_error(LOG_WARNING,
6786                     "Received invalid FMRI \"%s\" from repository server.\n",
6787                     fmri);
6788         } else if (inst_name != NULL && pg_name == NULL) {
6789                 (void) dgraph_remove_instance(fmri, h);
6790         } else if (inst_name != NULL && pg_name != NULL) {
6791                 /*
6792                  * If we're deleting the 'general' property group or
6793                  * 'general/enabled' property then the whole instance
6794                  * must be removed from the dgraph.
6795                  */
6796                 if (strcmp(pg_name, SCF_PG_GENERAL) != 0) {
6797                         free(lfmri);
6798                         return;
6799                 }
6800 
6801                 if (prop_name != NULL &&
6802                     strcmp(prop_name, SCF_PROPERTY_ENABLED) != 0) {
6803                         free(lfmri);
6804                         return;
6805                 }
6806 
6807                 /*
6808                  * Because the instance has already been deleted from the
6809                  * repository, we cannot use any scf_ functions to retrieve
6810                  * the instance FMRI however we can easily reconstruct it
6811                  * manually.
6812                  */
6813                 end_inst_fmri = strstr(fmri, SCF_FMRI_PROPERTYGRP_PREFIX);
6814                 if (end_inst_fmri == NULL)
6815                         bad_error("process_delete", 0);
6816 
6817                 end_inst_fmri[0] = '\0';
6818 
6819                 (void) dgraph_remove_instance(fmri, h);
6820         }
6821 
6822         free(lfmri);
6823 }
6824 
6825 /*ARGSUSED*/
6826 void *
6827 repository_event_thread(void *unused)
6828 {
6829         scf_handle_t *h;
6830         scf_propertygroup_t *pg;
6831         scf_instance_t *inst;
6832         char *fmri = startd_alloc(max_scf_fmri_size);
6833         char *pg_name = startd_alloc(max_scf_value_size);
6834         int r;
6835 
6836         h = libscf_handle_create_bound_loop();
6837 
6838         pg = safe_scf_pg_create(h);
6839         inst = safe_scf_instance_create(h);
6840 
6841 retry:
6842         if (_scf_notify_add_pgtype(h, SCF_GROUP_FRAMEWORK) != SCF_SUCCESS) {
6843                 if (scf_error() == SCF_ERROR_CONNECTION_BROKEN) {
6844                         libscf_handle_rebind(h);
6845                 } else {
6846                         log_error(LOG_WARNING,
6847                             "Couldn't set up repository notification "
6848                             "for property group type %s: %s\n",
6849                             SCF_GROUP_FRAMEWORK, scf_strerror(scf_error()));
6850 
6851                         (void) sleep(1);
6852                 }
6853 
6854                 goto retry;
6855         }
6856 
6857         /*CONSTCOND*/
6858         while (1) {
6859                 ssize_t res;
6860 
6861                 /* Note: fmri is only set on delete events. */
6862                 res = _scf_notify_wait(pg, fmri, max_scf_fmri_size);
6863                 if (res < 0) {
6864                         libscf_handle_rebind(h);
6865                         goto retry;
6866                 } else if (res == 0) {
6867                         /*
6868                          * property group modified.  inst and pg_name are
6869                          * pre-allocated scratch space.
6870                          */
6871                         if (scf_pg_update(pg) < 0) {
6872                                 switch (scf_error()) {
6873                                 case SCF_ERROR_DELETED:
6874                                         continue;
6875 
6876                                 case SCF_ERROR_CONNECTION_BROKEN:
6877                                         log_error(LOG_WARNING,
6878                                             "Lost repository event due to "
6879                                             "disconnection.\n");
6880                                         libscf_handle_rebind(h);
6881                                         goto retry;
6882 
6883                                 case SCF_ERROR_NOT_BOUND:
6884                                 case SCF_ERROR_NOT_SET:
6885                                 default:
6886                                         bad_error("scf_pg_update", scf_error());
6887                                 }
6888                         }
6889 
6890                         r = process_pg_event(h, pg, inst, pg_name);
6891                         switch (r) {
6892                         case 0:
6893                                 break;
6894 
6895                         case ECONNABORTED:
6896                                 log_error(LOG_WARNING, "Lost repository event "
6897                                     "due to disconnection.\n");
6898                                 libscf_handle_rebind(h);
6899                                 /* FALLTHROUGH */
6900 
6901                         case ECONNRESET:
6902                                 goto retry;
6903 
6904                         default:
6905                                 bad_error("process_pg_event", r);
6906                         }
6907                 } else {
6908                         /*
6909                          * Service, instance, or pg deleted.
6910                          * Don't trust fmri on return.
6911                          */
6912                         process_delete(fmri, h);
6913                 }
6914         }
6915 
6916         /*NOTREACHED*/
6917         return (NULL);
6918 }
6919 
6920 void
6921 graph_engine_start()
6922 {
6923         int err;
6924 
6925         (void) startd_thread_create(graph_thread, NULL);
6926 
6927         MUTEX_LOCK(&dgraph_lock);
6928         while (!initial_milestone_set) {
6929                 err = pthread_cond_wait(&initial_milestone_cv, &dgraph_lock);
6930                 assert(err == 0);
6931         }
6932         MUTEX_UNLOCK(&dgraph_lock);
6933 
6934         (void) startd_thread_create(repository_event_thread, NULL);
6935         (void) startd_thread_create(graph_event_thread, NULL);
6936 }
--- EOF ---