1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2018 Joyent, Inc.
  25  * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
  26  * Copyright 2016 Toomas Soome <tsoome@me.com>
  27  * Copyright 2016 RackTop Systems.
  28  */
  29 
  30 /*
  31  * graph.c - master restarter graph engine
  32  *
  33  *   The graph engine keeps a dependency graph of all service instances on the
  34  *   system, as recorded in the repository.  It decides when services should
  35  *   be brought up or down based on service states and dependencies and sends
  36  *   commands to restarters to effect any changes.  It also executes
  37  *   administrator commands sent by svcadm via the repository.
  38  *
  39  *   The graph is stored in uu_list_t *dgraph and its vertices are
  40  *   graph_vertex_t's, each of which has a name and an integer id unique to
  41  *   its name (see dict.c).  A vertex's type attribute designates the type
  42  *   of object it represents: GVT_INST for service instances, GVT_SVC for
  43  *   service objects (since service instances may depend on another service,
  44  *   rather than service instance), GVT_FILE for files (which services may
  45  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
  46  *   vertices are necessary because dependency lists may have particular
  47  *   grouping types (require any, require all, optional, or exclude) and
  48  *   event-propagation characteristics.
  49  *
  50  *   The initial graph is built by libscf_populate_graph() invoking
  51  *   dgraph_add_instance() for each instance in the repository.  The function
  52  *   adds a GVT_SVC vertex for the service if one does not already exist, adds
  53  *   a GVT_INST vertex named by the FMRI of the instance, and sets up the edges.
  54  *   The resulting web of vertices & edges associated with an instance's vertex
  55  *   includes
  56  *
  57  *     - an edge from the GVT_SVC vertex for the instance's service
  58  *
  59  *     - an edge to the GVT_INST vertex of the instance's resarter, if its
  60  *       restarter is not svc.startd
  61  *
  62  *     - edges from other GVT_INST vertices if the instance is a restarter
  63  *
  64  *     - for each dependency property group in the instance's "running"
  65  *       snapshot, an edge to a GVT_GROUP vertex named by the FMRI of the
  66  *       instance and the name of the property group
  67  *
  68  *     - for each value of the "entities" property in each dependency property
  69  *       group, an edge from the corresponding GVT_GROUP vertex to a
  70  *       GVT_INST, GVT_SVC, or GVT_FILE vertex
  71  *
  72  *     - edges from GVT_GROUP vertices for each dependent instance
  73  *
  74  *   After the edges are set up the vertex's GV_CONFIGURED flag is set.  If
  75  *   there are problems, or if a service is mentioned in a dependency but does
  76  *   not exist in the repository, the GV_CONFIGURED flag will be clear.
  77  *
  78  *   The graph and all of its vertices are protected by the dgraph_lock mutex.
  79  *   See restarter.c for more information.
  80  *
  81  *   The properties of an instance fall into two classes: immediate and
  82  *   snapshotted.  Immediate properties should have an immediate effect when
  83  *   changed.  Snapshotted properties should be read from a snapshot, so they
  84  *   only change when the snapshot changes.  The immediate properties used by
  85  *   the graph engine are general/enabled, general/restarter, and the properties
  86  *   in the restarter_actions property group.  Since they are immediate, they
  87  *   are not read out of a snapshot.  The snapshotted properties used by the
  88  *   graph engine are those in the property groups with type "dependency" and
  89  *   are read out of the "running" snapshot.  The "running" snapshot is created
  90  *   by the the graph engine as soon as possible, and it is updated, along with
  91  *   in-core copies of the data (dependency information for the graph engine) on
  92  *   receipt of the refresh command from svcadm.  In addition, the graph engine
  93  *   updates the "start" snapshot from the "running" snapshot whenever a service
  94  *   comes online.
  95  *
  96  *   When a DISABLE event is requested by the administrator, svc.startd shutdown
  97  *   the dependents first before shutting down the requested service.
  98  *   In graph_enable_by_vertex, we create a subtree that contains the dependent
  99  *   vertices by marking those vertices with the GV_TOOFFLINE flag. And we mark
 100  *   the vertex to disable with the GV_TODISABLE flag. Once the tree is created,
 101  *   we send the _ADMIN_DISABLE event to the leaves. The leaves will then
 102  *   transition from STATE_ONLINE/STATE_DEGRADED to STATE_OFFLINE/STATE_MAINT.
 103  *   In gt_enter_offline and gt_enter_maint if the vertex was in a subtree then
 104  *   we clear the GV_TOOFFLINE flag and walk the dependencies to offline the new
 105  *   exposed leaves. We do the same until we reach the last leaf (the one with
 106  *   the GV_TODISABLE flag). If the vertex to disable is also part of a larger
 107  *   subtree (eg. multiple DISABLE events on vertices in the same subtree) then
 108  *   once the first vertex is disabled (GV_TODISABLE flag is removed), we
 109  *   continue to propagate the offline event to the vertex's dependencies.
 110  *
 111  *
 112  * SMF state transition notifications
 113  *
 114  *   When an instance of a service managed by SMF changes state, svc.startd may
 115  *   publish a GPEC sysevent. All transitions to or from maintenance, a
 116  *   transition cause by a hardware error will generate an event.
 117  *   Other transitions will generate an event if there exist notification
 118  *   parameter for that transition. Notification parameters are stored in the
 119  *   SMF repository for the service/instance they refer to. System-wide
 120  *   notification parameters are stored in the global instance.
 121  *   svc.startd can be told to send events for all SMF state transitions despite
 122  *   of notification parameters by setting options/info_events_all to true in
 123  *   restarter:default
 124  *
 125  *   The set of transitions that generate events is cached in the
 126  *   dgraph_vertex_t gv_stn_tset for service/instance and in the global
 127  *   stn_global for the system-wide set. They are re-read when instances are
 128  *   refreshed.
 129  *
 130  *   The GPEC events published by svc.startd are consumed by fmd(1M). After
 131  *   processing these events, fmd(1M) publishes the processed events to
 132  *   notification agents. The notification agents read the notification
 133  *   parameters from the SMF repository through libscf(3LIB) interfaces and send
 134  *   the notification, or not, based on those parameters.
 135  *
 136  *   Subscription and publishing to the GPEC channels is done with the
 137  *   libfmevent(3LIB) wrappers fmev_[r]publish_*() and
 138  *   fmev_shdl_(un)subscribe().
 139  *
 140  */
 141 
 142 #include <sys/uadmin.h>
 143 #include <sys/wait.h>
 144 
 145 #include <assert.h>
 146 #include <errno.h>
 147 #include <fcntl.h>
 148 #include <fm/libfmevent.h>
 149 #include <libscf.h>
 150 #include <libscf_priv.h>
 151 #include <librestart.h>
 152 #include <libuutil.h>
 153 #include <locale.h>
 154 #include <poll.h>
 155 #include <pthread.h>
 156 #include <signal.h>
 157 #include <stddef.h>
 158 #include <stdio.h>
 159 #include <stdlib.h>
 160 #include <string.h>
 161 #include <strings.h>
 162 #include <sys/statvfs.h>
 163 #include <sys/uadmin.h>
 164 #include <zone.h>
 165 #if defined(__x86)
 166 #include <libbe.h>
 167 #endif  /* __x86 */
 168 
 169 #include "startd.h"
 170 #include "protocol.h"
 171 
 172 
 173 #define MILESTONE_NONE  ((graph_vertex_t *)1)
 174 
 175 #define CONSOLE_LOGIN_FMRI      "svc:/system/console-login:default"
 176 #define FS_MINIMAL_FMRI         "svc:/system/filesystem/minimal:default"
 177 
 178 #define VERTEX_REMOVED  0       /* vertex has been freed  */
 179 #define VERTEX_INUSE    1       /* vertex is still in use */
 180 
 181 #define IS_ENABLED(v) ((v)->gv_flags & (GV_ENABLED | GV_ENBLD_NOOVR))
 182 
 183 /*
 184  * stn_global holds the tset for the system wide notification parameters.
 185  * It is updated on refresh of svc:/system/svc/global:default
 186  *
 187  * There are two assumptions that relax the need for a mutex:
 188  *     1. 32-bit value assignments are atomic
 189  *     2. Its value is consumed only in one point at
 190  *     dgraph_state_transition_notify(). There are no test and set races.
 191  *
 192  *     If either assumption is broken, we'll need a mutex to synchronize
 193  *     access to stn_global
 194  */
 195 int32_t stn_global;
 196 /*
 197  * info_events_all holds a flag to override notification parameters and send
 198  * Information events for all state transitions.
 199  * same about the need of a mutex here.
 200  */
 201 int info_events_all;
 202 
 203 /*
 204  * Services in these states are not considered 'down' by the
 205  * milestone/shutdown code.
 206  */
 207 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
 208         (state) == RESTARTER_STATE_DEGRADED || \
 209         (state) == RESTARTER_STATE_OFFLINE)
 210 
 211 #define is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
 212         ((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \
 213         (v->gv_restart < RERR_RESTART)))
 214 
 215 #define is_inst_bypassed(v) ((v->gv_type == GVT_INST) && \
 216         ((v->gv_flags & GV_TODISABLE) || \
 217         (v->gv_flags & GV_TOOFFLINE)))
 218 
 219 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
 220 static uu_list_t *dgraph;
 221 static pthread_mutex_t dgraph_lock;
 222 
 223 /*
 224  * milestone indicates the current subgraph.  When NULL, it is the entire
 225  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
 226  * services on which the target vertex depends.
 227  */
 228 static graph_vertex_t *milestone = NULL;
 229 static boolean_t initial_milestone_set = B_FALSE;
 230 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
 231 
 232 /* protected by dgraph_lock */
 233 static boolean_t sulogin_thread_running = B_FALSE;
 234 static boolean_t sulogin_running = B_FALSE;
 235 static boolean_t console_login_ready = B_FALSE;
 236 
 237 /* Number of services to come down to complete milestone transition. */
 238 static uint_t non_subgraph_svcs;
 239 
 240 /*
 241  * These variables indicate what should be done when we reach the milestone
 242  * target milestone, i.e., when non_subgraph_svcs == 0.  They are acted upon in
 243  * dgraph_set_instance_state().
 244  */
 245 static int halting = -1;
 246 static boolean_t go_single_user_mode = B_FALSE;
 247 static boolean_t go_to_level1 = B_FALSE;
 248 
 249 /*
 250  * Tracks when we started halting.
 251  */
 252 static time_t halting_time = 0;
 253 
 254 /*
 255  * This tracks the legacy runlevel to ensure we signal init and manage
 256  * utmpx entries correctly.
 257  */
 258 static char current_runlevel = '\0';
 259 
 260 /* Number of single user threads currently running */
 261 static pthread_mutex_t single_user_thread_lock;
 262 static int single_user_thread_count = 0;
 263 
 264 /* Statistics for dependency cycle-checking */
 265 static u_longlong_t dep_inserts = 0;
 266 static u_longlong_t dep_cycle_ns = 0;
 267 static u_longlong_t dep_insert_ns = 0;
 268 
 269 
 270 static const char * const emsg_invalid_restarter =
 271         "Transitioning %s to maintenance, restarter FMRI %s is invalid "
 272         "(see 'svcs -xv' for details).\n";
 273 static const char * const console_login_fmri = CONSOLE_LOGIN_FMRI;
 274 static const char * const single_user_fmri = SCF_MILESTONE_SINGLE_USER;
 275 static const char * const multi_user_fmri = SCF_MILESTONE_MULTI_USER;
 276 static const char * const multi_user_svr_fmri = SCF_MILESTONE_MULTI_USER_SERVER;
 277 
 278 
 279 /*
 280  * These services define the system being "up".  If none of them can come
 281  * online, then we will run sulogin on the console.  Note that the install ones
 282  * are for the miniroot and when installing CDs after the first.  can_come_up()
 283  * does the decision making, and an sulogin_thread() runs sulogin, which can be
 284  * started by dgraph_set_instance_state() or single_user_thread().
 285  *
 286  * NOTE: can_come_up() relies on SCF_MILESTONE_SINGLE_USER being the first
 287  * entry, which is only used when booting_to_single_user (boot -s) is set.
 288  * This is because when doing a "boot -s", sulogin is started from specials.c
 289  * after milestone/single-user comes online, for backwards compatibility.
 290  * In this case, SCF_MILESTONE_SINGLE_USER needs to be part of up_svcs
 291  * to ensure sulogin will be spawned if milestone/single-user cannot be reached.
 292  */
 293 static const char * const up_svcs[] = {
 294         SCF_MILESTONE_SINGLE_USER,
 295         CONSOLE_LOGIN_FMRI,
 296         "svc:/system/install-setup:default",
 297         "svc:/system/install:default",
 298         NULL
 299 };
 300 
 301 /* This array must have an element for each non-NULL element of up_svcs[]. */
 302 static graph_vertex_t *up_svcs_p[] = { NULL, NULL, NULL, NULL };
 303 
 304 /* These are for seed repository magic.  See can_come_up(). */
 305 static const char * const manifest_import = SCF_INSTANCE_MI;
 306 static graph_vertex_t *manifest_import_p = NULL;
 307 
 308 
 309 static char target_milestone_as_runlevel(void);
 310 static void graph_runlevel_changed(char rl, int online);
 311 static int dgraph_set_milestone(const char *, scf_handle_t *, boolean_t);
 312 static boolean_t should_be_in_subgraph(graph_vertex_t *v);
 313 static int mark_subtree(graph_edge_t *, void *);
 314 static boolean_t insubtree_dependents_down(graph_vertex_t *);
 315 
 316 /*
 317  * graph_vertex_compare()
 318  *      This function can compare either int *id or * graph_vertex_t *gv
 319  *      values, as the vertex id is always the first element of a
 320  *      graph_vertex structure.
 321  */
 322 /* ARGSUSED */
 323 static int
 324 graph_vertex_compare(const void *lc_arg, const void *rc_arg, void *private)
 325 {
 326         int lc_id = ((const graph_vertex_t *)lc_arg)->gv_id;
 327         int rc_id = *(int *)rc_arg;
 328 
 329         if (lc_id > rc_id)
 330                 return (1);
 331         if (lc_id < rc_id)
 332                 return (-1);
 333         return (0);
 334 }
 335 
 336 void
 337 graph_init()
 338 {
 339         graph_edge_pool = startd_list_pool_create("graph_edges",
 340             sizeof (graph_edge_t), offsetof(graph_edge_t, ge_link), NULL,
 341             UU_LIST_POOL_DEBUG);
 342         assert(graph_edge_pool != NULL);
 343 
 344         graph_vertex_pool = startd_list_pool_create("graph_vertices",
 345             sizeof (graph_vertex_t), offsetof(graph_vertex_t, gv_link),
 346             graph_vertex_compare, UU_LIST_POOL_DEBUG);
 347         assert(graph_vertex_pool != NULL);
 348 
 349         (void) pthread_mutex_init(&dgraph_lock, &mutex_attrs);
 350         (void) pthread_mutex_init(&single_user_thread_lock, &mutex_attrs);
 351         dgraph = startd_list_create(graph_vertex_pool, NULL, UU_LIST_SORTED);
 352         assert(dgraph != NULL);
 353 
 354         if (!st->st_initial)
 355                 current_runlevel = utmpx_get_runlevel();
 356 
 357         log_framework(LOG_DEBUG, "Initialized graph\n");
 358 }
 359 
 360 static graph_vertex_t *
 361 vertex_get_by_name(const char *name)
 362 {
 363         int id;
 364 
 365         assert(MUTEX_HELD(&dgraph_lock));
 366 
 367         id = dict_lookup_byname(name);
 368         if (id == -1)
 369                 return (NULL);
 370 
 371         return (uu_list_find(dgraph, &id, NULL, NULL));
 372 }
 373 
 374 static graph_vertex_t *
 375 vertex_get_by_id(int id)
 376 {
 377         assert(MUTEX_HELD(&dgraph_lock));
 378 
 379         if (id == -1)
 380                 return (NULL);
 381 
 382         return (uu_list_find(dgraph, &id, NULL, NULL));
 383 }
 384 
 385 /*
 386  * Creates a new vertex with the given name, adds it to the graph, and returns
 387  * a pointer to it.  The graph lock must be held by this thread on entry.
 388  */
 389 static graph_vertex_t *
 390 graph_add_vertex(const char *name)
 391 {
 392         int id;
 393         graph_vertex_t *v;
 394         void *p;
 395         uu_list_index_t idx;
 396 
 397         assert(MUTEX_HELD(&dgraph_lock));
 398 
 399         id = dict_insert(name);
 400 
 401         v = startd_zalloc(sizeof (*v));
 402 
 403         v->gv_id = id;
 404 
 405         v->gv_name = startd_alloc(strlen(name) + 1);
 406         (void) strcpy(v->gv_name, name);
 407 
 408         v->gv_dependencies = startd_list_create(graph_edge_pool, v, 0);
 409         v->gv_dependents = startd_list_create(graph_edge_pool, v, 0);
 410 
 411         p = uu_list_find(dgraph, &id, NULL, &idx);
 412         assert(p == NULL);
 413 
 414         uu_list_node_init(v, &v->gv_link, graph_vertex_pool);
 415         uu_list_insert(dgraph, v, idx);
 416 
 417         return (v);
 418 }
 419 
 420 /*
 421  * Removes v from the graph and frees it.  The graph should be locked by this
 422  * thread, and v should have no edges associated with it.
 423  */
 424 static void
 425 graph_remove_vertex(graph_vertex_t *v)
 426 {
 427         assert(MUTEX_HELD(&dgraph_lock));
 428 
 429         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 430         assert(uu_list_numnodes(v->gv_dependents) == 0);
 431         assert(v->gv_refs == 0);
 432 
 433         startd_free(v->gv_name, strlen(v->gv_name) + 1);
 434         uu_list_destroy(v->gv_dependencies);
 435         uu_list_destroy(v->gv_dependents);
 436         uu_list_remove(dgraph, v);
 437 
 438         startd_free(v, sizeof (graph_vertex_t));
 439 }
 440 
 441 static void
 442 graph_add_edge(graph_vertex_t *fv, graph_vertex_t *tv)
 443 {
 444         graph_edge_t *e, *re;
 445         int r;
 446 
 447         assert(MUTEX_HELD(&dgraph_lock));
 448 
 449         e = startd_alloc(sizeof (graph_edge_t));
 450         re = startd_alloc(sizeof (graph_edge_t));
 451 
 452         e->ge_parent = fv;
 453         e->ge_vertex = tv;
 454 
 455         re->ge_parent = tv;
 456         re->ge_vertex = fv;
 457 
 458         uu_list_node_init(e, &e->ge_link, graph_edge_pool);
 459         r = uu_list_insert_before(fv->gv_dependencies, NULL, e);
 460         assert(r == 0);
 461 
 462         uu_list_node_init(re, &re->ge_link, graph_edge_pool);
 463         r = uu_list_insert_before(tv->gv_dependents, NULL, re);
 464         assert(r == 0);
 465 }
 466 
 467 static void
 468 graph_remove_edge(graph_vertex_t *v, graph_vertex_t *dv)
 469 {
 470         graph_edge_t *e;
 471 
 472         for (e = uu_list_first(v->gv_dependencies);
 473             e != NULL;
 474             e = uu_list_next(v->gv_dependencies, e)) {
 475                 if (e->ge_vertex == dv) {
 476                         uu_list_remove(v->gv_dependencies, e);
 477                         startd_free(e, sizeof (graph_edge_t));
 478                         break;
 479                 }
 480         }
 481 
 482         for (e = uu_list_first(dv->gv_dependents);
 483             e != NULL;
 484             e = uu_list_next(dv->gv_dependents, e)) {
 485                 if (e->ge_vertex == v) {
 486                         uu_list_remove(dv->gv_dependents, e);
 487                         startd_free(e, sizeof (graph_edge_t));
 488                         break;
 489                 }
 490         }
 491 }
 492 
 493 static void
 494 remove_inst_vertex(graph_vertex_t *v)
 495 {
 496         graph_edge_t *e;
 497         graph_vertex_t *sv;
 498         int i;
 499 
 500         assert(MUTEX_HELD(&dgraph_lock));
 501         assert(uu_list_numnodes(v->gv_dependents) == 1);
 502         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 503         assert(v->gv_refs == 0);
 504         assert((v->gv_flags & GV_CONFIGURED) == 0);
 505 
 506         e = uu_list_first(v->gv_dependents);
 507         sv = e->ge_vertex;
 508         graph_remove_edge(sv, v);
 509 
 510         for (i = 0; up_svcs[i] != NULL; ++i) {
 511                 if (up_svcs_p[i] == v)
 512                         up_svcs_p[i] = NULL;
 513         }
 514 
 515         if (manifest_import_p == v)
 516                 manifest_import_p = NULL;
 517 
 518         graph_remove_vertex(v);
 519 
 520         if (uu_list_numnodes(sv->gv_dependencies) == 0 &&
 521             uu_list_numnodes(sv->gv_dependents) == 0 &&
 522             sv->gv_refs == 0)
 523                 graph_remove_vertex(sv);
 524 }
 525 
 526 static void
 527 graph_walk_dependents(graph_vertex_t *v, void (*func)(graph_vertex_t *, void *),
 528     void *arg)
 529 {
 530         graph_edge_t *e;
 531 
 532         for (e = uu_list_first(v->gv_dependents);
 533             e != NULL;
 534             e = uu_list_next(v->gv_dependents, e))
 535                 func(e->ge_vertex, arg);
 536 }
 537 
 538 static void
 539 graph_walk_dependencies(graph_vertex_t *v,
 540     void (*func)(graph_vertex_t *, void *), void *arg)
 541 {
 542         graph_edge_t *e;
 543 
 544         assert(MUTEX_HELD(&dgraph_lock));
 545 
 546         for (e = uu_list_first(v->gv_dependencies);
 547             e != NULL;
 548             e = uu_list_next(v->gv_dependencies, e)) {
 549 
 550                 func(e->ge_vertex, arg);
 551         }
 552 }
 553 
 554 /*
 555  * Generic graph walking function.
 556  *
 557  * Given a vertex, this function will walk either dependencies
 558  * (WALK_DEPENDENCIES) or dependents (WALK_DEPENDENTS) of a vertex recursively
 559  * for the entire graph.  It will avoid cycles and never visit the same vertex
 560  * twice.
 561  *
 562  * We avoid traversing exclusion dependencies, because they are allowed to
 563  * create cycles in the graph.  When propagating satisfiability, there is no
 564  * need to walk exclusion dependencies because exclude_all_satisfied() doesn't
 565  * test for satisfiability.
 566  *
 567  * The walker takes two callbacks.  The first is called before examining the
 568  * dependents of each vertex.  The second is called on each vertex after
 569  * examining its dependents.  This allows is_path_to() to construct a path only
 570  * after the target vertex has been found.
 571  */
 572 typedef enum {
 573         WALK_DEPENDENTS,
 574         WALK_DEPENDENCIES
 575 } graph_walk_dir_t;
 576 
 577 typedef int (*graph_walk_cb_t)(graph_vertex_t *, void *);
 578 
 579 typedef struct graph_walk_info {
 580         graph_walk_dir_t        gi_dir;
 581         uchar_t                 *gi_visited;    /* vertex bitmap */
 582         int                     (*gi_pre)(graph_vertex_t *, void *);
 583         void                    (*gi_post)(graph_vertex_t *, void *);
 584         void                    *gi_arg;        /* callback arg */
 585         int                     gi_ret;         /* return value */
 586 } graph_walk_info_t;
 587 
 588 static int
 589 graph_walk_recurse(graph_edge_t *e, graph_walk_info_t *gip)
 590 {
 591         uu_list_t *list;
 592         int r;
 593         graph_vertex_t *v = e->ge_vertex;
 594         int i;
 595         uint_t b;
 596 
 597         i = v->gv_id / 8;
 598         b = 1 << (v->gv_id % 8);
 599 
 600         /*
 601          * Check to see if we've visited this vertex already.
 602          */
 603         if (gip->gi_visited[i] & b)
 604                 return (UU_WALK_NEXT);
 605 
 606         gip->gi_visited[i] |= b;
 607 
 608         /*
 609          * Don't follow exclusions.
 610          */
 611         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
 612                 return (UU_WALK_NEXT);
 613 
 614         /*
 615          * Call pre-visit callback.  If this doesn't terminate the walk,
 616          * continue search.
 617          */
 618         if ((gip->gi_ret = gip->gi_pre(v, gip->gi_arg)) == UU_WALK_NEXT) {
 619                 /*
 620                  * Recurse using appropriate list.
 621                  */
 622                 if (gip->gi_dir == WALK_DEPENDENTS)
 623                         list = v->gv_dependents;
 624                 else
 625                         list = v->gv_dependencies;
 626 
 627                 r = uu_list_walk(list, (uu_walk_fn_t *)graph_walk_recurse,
 628                     gip, 0);
 629                 assert(r == 0);
 630         }
 631 
 632         /*
 633          * Callbacks must return either UU_WALK_NEXT or UU_WALK_DONE.
 634          */
 635         assert(gip->gi_ret == UU_WALK_NEXT || gip->gi_ret == UU_WALK_DONE);
 636 
 637         /*
 638          * If given a post-callback, call the function for every vertex.
 639          */
 640         if (gip->gi_post != NULL)
 641                 (void) gip->gi_post(v, gip->gi_arg);
 642 
 643         /*
 644          * Preserve the callback's return value.  If the callback returns
 645          * UU_WALK_DONE, then we propagate that to the caller in order to
 646          * terminate the walk.
 647          */
 648         return (gip->gi_ret);
 649 }
 650 
 651 static void
 652 graph_walk(graph_vertex_t *v, graph_walk_dir_t dir,
 653     int (*pre)(graph_vertex_t *, void *),
 654     void (*post)(graph_vertex_t *, void *), void *arg)
 655 {
 656         graph_walk_info_t gi;
 657         graph_edge_t fake;
 658         size_t sz = dictionary->dict_new_id / 8 + 1;
 659 
 660         gi.gi_visited = startd_zalloc(sz);
 661         gi.gi_pre = pre;
 662         gi.gi_post = post;
 663         gi.gi_arg = arg;
 664         gi.gi_dir = dir;
 665         gi.gi_ret = 0;
 666 
 667         /*
 668          * Fake up an edge for the first iteration
 669          */
 670         fake.ge_vertex = v;
 671         (void) graph_walk_recurse(&fake, &gi);
 672 
 673         startd_free(gi.gi_visited, sz);
 674 }
 675 
 676 typedef struct child_search {
 677         int     id;             /* id of vertex to look for */
 678         uint_t  depth;          /* recursion depth */
 679         /*
 680          * While the vertex is not found, path is NULL.  After the search, if
 681          * the vertex was found then path should point to a -1-terminated
 682          * array of vertex id's which constitute the path to the vertex.
 683          */
 684         int     *path;
 685 } child_search_t;
 686 
 687 static int
 688 child_pre(graph_vertex_t *v, void *arg)
 689 {
 690         child_search_t *cs = arg;
 691 
 692         cs->depth++;
 693 
 694         if (v->gv_id == cs->id) {
 695                 cs->path = startd_alloc((cs->depth + 1) * sizeof (int));
 696                 cs->path[cs->depth] = -1;
 697                 return (UU_WALK_DONE);
 698         }
 699 
 700         return (UU_WALK_NEXT);
 701 }
 702 
 703 static void
 704 child_post(graph_vertex_t *v, void *arg)
 705 {
 706         child_search_t *cs = arg;
 707 
 708         cs->depth--;
 709 
 710         if (cs->path != NULL)
 711                 cs->path[cs->depth] = v->gv_id;
 712 }
 713 
 714 /*
 715  * Look for a path from from to to.  If one exists, returns a pointer to
 716  * a NULL-terminated array of pointers to the vertices along the path.  If
 717  * there is no path, returns NULL.
 718  */
 719 static int *
 720 is_path_to(graph_vertex_t *from, graph_vertex_t *to)
 721 {
 722         child_search_t cs;
 723 
 724         cs.id = to->gv_id;
 725         cs.depth = 0;
 726         cs.path = NULL;
 727 
 728         graph_walk(from, WALK_DEPENDENCIES, child_pre, child_post, &cs);
 729 
 730         return (cs.path);
 731 }
 732 
 733 /*
 734  * Given an array of int's as returned by is_path_to, allocates a string of
 735  * their names joined by newlines.  Returns the size of the allocated buffer
 736  * in *sz and frees path.
 737  */
 738 static void
 739 path_to_str(int *path, char **cpp, size_t *sz)
 740 {
 741         int i;
 742         graph_vertex_t *v;
 743         size_t allocd, new_allocd;
 744         char *new, *name;
 745 
 746         assert(MUTEX_HELD(&dgraph_lock));
 747         assert(path[0] != -1);
 748 
 749         allocd = 1;
 750         *cpp = startd_alloc(1);
 751         (*cpp)[0] = '\0';
 752 
 753         for (i = 0; path[i] != -1; ++i) {
 754                 name = NULL;
 755 
 756                 v = vertex_get_by_id(path[i]);
 757 
 758                 if (v == NULL)
 759                         name = "<deleted>";
 760                 else if (v->gv_type == GVT_INST || v->gv_type == GVT_SVC)
 761                         name = v->gv_name;
 762 
 763                 if (name != NULL) {
 764                         new_allocd = allocd + strlen(name) + 1;
 765                         new = startd_alloc(new_allocd);
 766                         (void) strcpy(new, *cpp);
 767                         (void) strcat(new, name);
 768                         (void) strcat(new, "\n");
 769 
 770                         startd_free(*cpp, allocd);
 771 
 772                         *cpp = new;
 773                         allocd = new_allocd;
 774                 }
 775         }
 776 
 777         startd_free(path, sizeof (int) * (i + 1));
 778 
 779         *sz = allocd;
 780 }
 781 
 782 
 783 /*
 784  * This function along with run_sulogin() implements an exclusion relationship
 785  * between system/console-login and sulogin.  run_sulogin() will fail if
 786  * system/console-login is online, and the graph engine should call
 787  * graph_clogin_start() to bring system/console-login online, which defers the
 788  * start if sulogin is running.
 789  */
 790 static void
 791 graph_clogin_start(graph_vertex_t *v)
 792 {
 793         assert(MUTEX_HELD(&dgraph_lock));
 794 
 795         if (sulogin_running)
 796                 console_login_ready = B_TRUE;
 797         else
 798                 vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 799 }
 800 
 801 static void
 802 graph_su_start(graph_vertex_t *v)
 803 {
 804         /*
 805          * /etc/inittab used to have the initial /sbin/rcS as a 'sysinit'
 806          * entry with a runlevel of 'S', before jumping to the final
 807          * target runlevel (as set in initdefault).  We mimic that legacy
 808          * behavior here.
 809          */
 810         utmpx_set_runlevel('S', '0', B_FALSE);
 811         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 812 }
 813 
 814 static void
 815 graph_post_su_online(void)
 816 {
 817         graph_runlevel_changed('S', 1);
 818 }
 819 
 820 static void
 821 graph_post_su_disable(void)
 822 {
 823         graph_runlevel_changed('S', 0);
 824 }
 825 
 826 static void
 827 graph_post_mu_online(void)
 828 {
 829         graph_runlevel_changed('2', 1);
 830 }
 831 
 832 static void
 833 graph_post_mu_disable(void)
 834 {
 835         graph_runlevel_changed('2', 0);
 836 }
 837 
 838 static void
 839 graph_post_mus_online(void)
 840 {
 841         graph_runlevel_changed('3', 1);
 842 }
 843 
 844 static void
 845 graph_post_mus_disable(void)
 846 {
 847         graph_runlevel_changed('3', 0);
 848 }
 849 
 850 static struct special_vertex_info {
 851         const char      *name;
 852         void            (*start_f)(graph_vertex_t *);
 853         void            (*post_online_f)(void);
 854         void            (*post_disable_f)(void);
 855 } special_vertices[] = {
 856         { CONSOLE_LOGIN_FMRI, graph_clogin_start, NULL, NULL },
 857         { SCF_MILESTONE_SINGLE_USER, graph_su_start,
 858             graph_post_su_online, graph_post_su_disable },
 859         { SCF_MILESTONE_MULTI_USER, NULL,
 860             graph_post_mu_online, graph_post_mu_disable },
 861         { SCF_MILESTONE_MULTI_USER_SERVER, NULL,
 862             graph_post_mus_online, graph_post_mus_disable },
 863         { NULL },
 864 };
 865 
 866 
 867 void
 868 vertex_send_event(graph_vertex_t *v, restarter_event_type_t e)
 869 {
 870         switch (e) {
 871         case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
 872                 assert(v->gv_state == RESTARTER_STATE_UNINIT);
 873 
 874                 MUTEX_LOCK(&st->st_load_lock);
 875                 st->st_load_instances++;
 876                 MUTEX_UNLOCK(&st->st_load_lock);
 877                 break;
 878 
 879         case RESTARTER_EVENT_TYPE_ENABLE:
 880                 log_framework(LOG_DEBUG, "Enabling %s.\n", v->gv_name);
 881                 assert(v->gv_state == RESTARTER_STATE_UNINIT ||
 882                     v->gv_state == RESTARTER_STATE_DISABLED ||
 883                     v->gv_state == RESTARTER_STATE_MAINT);
 884                 break;
 885 
 886         case RESTARTER_EVENT_TYPE_DISABLE:
 887         case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
 888                 log_framework(LOG_DEBUG, "Disabling %s.\n", v->gv_name);
 889                 assert(v->gv_state != RESTARTER_STATE_DISABLED);
 890                 break;
 891 
 892         case RESTARTER_EVENT_TYPE_STOP_RESET:
 893         case RESTARTER_EVENT_TYPE_STOP:
 894                 log_framework(LOG_DEBUG, "Stopping %s.\n", v->gv_name);
 895                 assert(v->gv_state == RESTARTER_STATE_DEGRADED ||
 896                     v->gv_state == RESTARTER_STATE_ONLINE);
 897                 break;
 898 
 899         case RESTARTER_EVENT_TYPE_START:
 900                 log_framework(LOG_DEBUG, "Starting %s.\n", v->gv_name);
 901                 assert(v->gv_state == RESTARTER_STATE_OFFLINE);
 902                 break;
 903 
 904         case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
 905         case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
 906         case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
 907         case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
 908         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
 909         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
 910         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
 911         case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
 912         case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
 913                 break;
 914 
 915         default:
 916 #ifndef NDEBUG
 917                 uu_warn("%s:%d: Bad event %d.\n", __FILE__, __LINE__, e);
 918 #endif
 919                 abort();
 920         }
 921 
 922         restarter_protocol_send_event(v->gv_name, v->gv_restarter_channel, e,
 923             v->gv_reason);
 924 }
 925 
 926 static void
 927 graph_unset_restarter(graph_vertex_t *v)
 928 {
 929         assert(MUTEX_HELD(&dgraph_lock));
 930         assert(v->gv_flags & GV_CONFIGURED);
 931 
 932         vertex_send_event(v, RESTARTER_EVENT_TYPE_REMOVE_INSTANCE);
 933 
 934         if (v->gv_restarter_id != -1) {
 935                 graph_vertex_t *rv;
 936 
 937                 rv = vertex_get_by_id(v->gv_restarter_id);
 938                 graph_remove_edge(v, rv);
 939         }
 940 
 941         v->gv_restarter_id = -1;
 942         v->gv_restarter_channel = NULL;
 943 }
 944 
 945 /*
 946  * Return VERTEX_REMOVED when the vertex passed in argument is deleted from the
 947  * dgraph otherwise return VERTEX_INUSE.
 948  */
 949 static int
 950 free_if_unrefed(graph_vertex_t *v)
 951 {
 952         assert(MUTEX_HELD(&dgraph_lock));
 953 
 954         if (v->gv_refs > 0)
 955                 return (VERTEX_INUSE);
 956 
 957         if (v->gv_type == GVT_SVC &&
 958             uu_list_numnodes(v->gv_dependents) == 0 &&
 959             uu_list_numnodes(v->gv_dependencies) == 0) {
 960                 graph_remove_vertex(v);
 961                 return (VERTEX_REMOVED);
 962         } else if (v->gv_type == GVT_INST &&
 963             (v->gv_flags & GV_CONFIGURED) == 0 &&
 964             uu_list_numnodes(v->gv_dependents) == 1 &&
 965             uu_list_numnodes(v->gv_dependencies) == 0) {
 966                 remove_inst_vertex(v);
 967                 return (VERTEX_REMOVED);
 968         }
 969 
 970         return (VERTEX_INUSE);
 971 }
 972 
 973 static void
 974 delete_depgroup(graph_vertex_t *v)
 975 {
 976         graph_edge_t *e;
 977         graph_vertex_t *dv;
 978 
 979         assert(MUTEX_HELD(&dgraph_lock));
 980         assert(v->gv_type == GVT_GROUP);
 981         assert(uu_list_numnodes(v->gv_dependents) == 0);
 982 
 983         while ((e = uu_list_first(v->gv_dependencies)) != NULL) {
 984                 dv = e->ge_vertex;
 985 
 986                 graph_remove_edge(v, dv);
 987 
 988                 switch (dv->gv_type) {
 989                 case GVT_INST:          /* instance dependency */
 990                 case GVT_SVC:           /* service dependency */
 991                         (void) free_if_unrefed(dv);
 992                         break;
 993 
 994                 case GVT_FILE:          /* file dependency */
 995                         assert(uu_list_numnodes(dv->gv_dependencies) == 0);
 996                         if (uu_list_numnodes(dv->gv_dependents) == 0)
 997                                 graph_remove_vertex(dv);
 998                         break;
 999 
1000                 default:
1001 #ifndef NDEBUG
1002                         uu_warn("%s:%d: Unexpected node type %d", __FILE__,
1003                             __LINE__, dv->gv_type);
1004 #endif
1005                         abort();
1006                 }
1007         }
1008 
1009         graph_remove_vertex(v);
1010 }
1011 
1012 static int
1013 delete_instance_deps_cb(graph_edge_t *e, void **ptrs)
1014 {
1015         graph_vertex_t *v = ptrs[0];
1016         boolean_t delete_restarter_dep = (boolean_t)ptrs[1];
1017         graph_vertex_t *dv;
1018 
1019         dv = e->ge_vertex;
1020 
1021         /*
1022          * We have four possibilities here:
1023          *   - GVT_INST: restarter
1024          *   - GVT_GROUP - GVT_INST: instance dependency
1025          *   - GVT_GROUP - GVT_SVC - GV_INST: service dependency
1026          *   - GVT_GROUP - GVT_FILE: file dependency
1027          */
1028         switch (dv->gv_type) {
1029         case GVT_INST:  /* restarter */
1030                 assert(dv->gv_id == v->gv_restarter_id);
1031                 if (delete_restarter_dep)
1032                         graph_remove_edge(v, dv);
1033                 break;
1034 
1035         case GVT_GROUP: /* pg dependency */
1036                 graph_remove_edge(v, dv);
1037                 delete_depgroup(dv);
1038                 break;
1039 
1040         case GVT_FILE:
1041                 /* These are currently not direct dependencies */
1042 
1043         default:
1044 #ifndef NDEBUG
1045                 uu_warn("%s:%d: Bad vertex type %d.\n", __FILE__, __LINE__,
1046                     dv->gv_type);
1047 #endif
1048                 abort();
1049         }
1050 
1051         return (UU_WALK_NEXT);
1052 }
1053 
1054 static void
1055 delete_instance_dependencies(graph_vertex_t *v, boolean_t delete_restarter_dep)
1056 {
1057         void *ptrs[2];
1058         int r;
1059 
1060         assert(MUTEX_HELD(&dgraph_lock));
1061         assert(v->gv_type == GVT_INST);
1062 
1063         ptrs[0] = v;
1064         ptrs[1] = (void *)delete_restarter_dep;
1065 
1066         r = uu_list_walk(v->gv_dependencies,
1067             (uu_walk_fn_t *)delete_instance_deps_cb, &ptrs, UU_WALK_ROBUST);
1068         assert(r == 0);
1069 }
1070 
1071 /*
1072  * int graph_insert_vertex_unconfigured()
1073  *   Insert a vertex without sending any restarter events. If the vertex
1074  *   already exists or creation is successful, return a pointer to it in *vp.
1075  *
1076  *   If type is not GVT_GROUP, dt can remain unset.
1077  *
1078  *   Returns 0, EEXIST, or EINVAL if the arguments are invalid (i.e., fmri
1079  *   doesn't agree with type, or type doesn't agree with dt).
1080  */
1081 static int
1082 graph_insert_vertex_unconfigured(const char *fmri, gv_type_t type,
1083     depgroup_type_t dt, restarter_error_t rt, graph_vertex_t **vp)
1084 {
1085         int r;
1086         int i;
1087 
1088         assert(MUTEX_HELD(&dgraph_lock));
1089 
1090         switch (type) {
1091         case GVT_SVC:
1092         case GVT_INST:
1093                 if (strncmp(fmri, "svc:", sizeof ("svc:") - 1) != 0)
1094                         return (EINVAL);
1095                 break;
1096 
1097         case GVT_FILE:
1098                 if (strncmp(fmri, "file:", sizeof ("file:") - 1) != 0)
1099                         return (EINVAL);
1100                 break;
1101 
1102         case GVT_GROUP:
1103                 if (dt <= 0 || rt < 0)
1104                         return (EINVAL);
1105                 break;
1106 
1107         default:
1108 #ifndef NDEBUG
1109                 uu_warn("%s:%d: Unknown type %d.\n", __FILE__, __LINE__, type);
1110 #endif
1111                 abort();
1112         }
1113 
1114         *vp = vertex_get_by_name(fmri);
1115         if (*vp != NULL)
1116                 return (EEXIST);
1117 
1118         *vp = graph_add_vertex(fmri);
1119 
1120         (*vp)->gv_type = type;
1121         (*vp)->gv_depgroup = dt;
1122         (*vp)->gv_restart = rt;
1123 
1124         (*vp)->gv_flags = 0;
1125         (*vp)->gv_state = RESTARTER_STATE_NONE;
1126 
1127         for (i = 0; special_vertices[i].name != NULL; ++i) {
1128                 if (strcmp(fmri, special_vertices[i].name) == 0) {
1129                         (*vp)->gv_start_f = special_vertices[i].start_f;
1130                         (*vp)->gv_post_online_f =
1131                             special_vertices[i].post_online_f;
1132                         (*vp)->gv_post_disable_f =
1133                             special_vertices[i].post_disable_f;
1134                         break;
1135                 }
1136         }
1137 
1138         (*vp)->gv_restarter_id = -1;
1139         (*vp)->gv_restarter_channel = 0;
1140 
1141         if (type == GVT_INST) {
1142                 char *sfmri;
1143                 graph_vertex_t *sv;
1144 
1145                 sfmri = inst_fmri_to_svc_fmri(fmri);
1146                 sv = vertex_get_by_name(sfmri);
1147                 if (sv == NULL) {
1148                         r = graph_insert_vertex_unconfigured(sfmri, GVT_SVC, 0,
1149                             0, &sv);
1150                         assert(r == 0);
1151                 }
1152                 startd_free(sfmri, max_scf_fmri_size);
1153 
1154                 graph_add_edge(sv, *vp);
1155         }
1156 
1157         /*
1158          * If this vertex is in the subgraph, mark it as so, for both
1159          * GVT_INST and GVT_SERVICE verteces.
1160          * A GVT_SERVICE vertex can only be in the subgraph if another instance
1161          * depends on it, in which case it's already been added to the graph
1162          * and marked as in the subgraph (by refresh_vertex()).  If a
1163          * GVT_SERVICE vertex was freshly added (by the code above), it means
1164          * that it has no dependents, and cannot be in the subgraph.
1165          * Regardless of this, we still check that gv_flags includes
1166          * GV_INSUBGRAPH in the event that future behavior causes the above
1167          * code to add a GVT_SERVICE vertex which should be in the subgraph.
1168          */
1169 
1170         (*vp)->gv_flags |= (should_be_in_subgraph(*vp)? GV_INSUBGRAPH : 0);
1171 
1172         return (0);
1173 }
1174 
1175 /*
1176  * Returns 0 on success or ELOOP if the dependency would create a cycle.
1177  */
1178 static int
1179 graph_insert_dependency(graph_vertex_t *fv, graph_vertex_t *tv, int **pathp)
1180 {
1181         hrtime_t now;
1182 
1183         assert(MUTEX_HELD(&dgraph_lock));
1184 
1185         /* cycle detection */
1186         now = gethrtime();
1187 
1188         /* Don't follow exclusions. */
1189         if (!(fv->gv_type == GVT_GROUP &&
1190             fv->gv_depgroup == DEPGRP_EXCLUDE_ALL)) {
1191                 *pathp = is_path_to(tv, fv);
1192                 if (*pathp)
1193                         return (ELOOP);
1194         }
1195 
1196         dep_cycle_ns += gethrtime() - now;
1197         ++dep_inserts;
1198         now = gethrtime();
1199 
1200         graph_add_edge(fv, tv);
1201 
1202         dep_insert_ns += gethrtime() - now;
1203 
1204         /* Check if the dependency adds the "to" vertex to the subgraph */
1205         tv->gv_flags |= (should_be_in_subgraph(tv) ? GV_INSUBGRAPH : 0);
1206 
1207         return (0);
1208 }
1209 
1210 static int
1211 inst_running(graph_vertex_t *v)
1212 {
1213         assert(v->gv_type == GVT_INST);
1214 
1215         if (v->gv_state == RESTARTER_STATE_ONLINE ||
1216             v->gv_state == RESTARTER_STATE_DEGRADED)
1217                 return (1);
1218 
1219         return (0);
1220 }
1221 
1222 /*
1223  * The dependency evaluation functions return
1224  *   1 - dependency satisfied
1225  *   0 - dependency unsatisfied
1226  *   -1 - dependency unsatisfiable (without administrator intervention)
1227  *
1228  * The functions also take a boolean satbility argument.  When true, the
1229  * functions may recurse in order to determine satisfiability.
1230  */
1231 static int require_any_satisfied(graph_vertex_t *, boolean_t);
1232 static int dependency_satisfied(graph_vertex_t *, boolean_t);
1233 
1234 /*
1235  * A require_all dependency is unsatisfied if any elements are unsatisfied.  It
1236  * is unsatisfiable if any elements are unsatisfiable.
1237  */
1238 static int
1239 require_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1240 {
1241         graph_edge_t *edge;
1242         int i;
1243         boolean_t any_unsatisfied;
1244 
1245         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1246                 return (1);
1247 
1248         any_unsatisfied = B_FALSE;
1249 
1250         for (edge = uu_list_first(groupv->gv_dependencies);
1251             edge != NULL;
1252             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1253                 i = dependency_satisfied(edge->ge_vertex, satbility);
1254                 if (i == 1)
1255                         continue;
1256 
1257                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1258                     "require_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1259                     edge->ge_vertex->gv_name, i == 0 ? "ed" : "able");
1260 
1261                 if (!satbility)
1262                         return (0);
1263 
1264                 if (i == -1)
1265                         return (-1);
1266 
1267                 any_unsatisfied = B_TRUE;
1268         }
1269 
1270         return (any_unsatisfied ? 0 : 1);
1271 }
1272 
1273 /*
1274  * A require_any dependency is satisfied if any element is satisfied.  It is
1275  * satisfiable if any element is satisfiable.
1276  */
1277 static int
1278 require_any_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1279 {
1280         graph_edge_t *edge;
1281         int s;
1282         boolean_t satisfiable;
1283 
1284         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1285                 return (1);
1286 
1287         satisfiable = B_FALSE;
1288 
1289         for (edge = uu_list_first(groupv->gv_dependencies);
1290             edge != NULL;
1291             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1292                 s = dependency_satisfied(edge->ge_vertex, satbility);
1293 
1294                 if (s == 1)
1295                         return (1);
1296 
1297                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1298                     "require_any(%s): %s is unsatisfi%s.\n",
1299                     groupv->gv_name, edge->ge_vertex->gv_name,
1300                     s == 0 ? "ed" : "able");
1301 
1302                 if (satbility && s == 0)
1303                         satisfiable = B_TRUE;
1304         }
1305 
1306         return ((!satbility || satisfiable) ? 0 : -1);
1307 }
1308 
1309 /*
1310  * An optional_all dependency only considers elements which are configured,
1311  * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1312  * is unsatisfied.
1313  *
1314  * Offline dependencies which are waiting for a dependency to come online are
1315  * unsatisfied.  Offline dependences which cannot possibly come online
1316  * (unsatisfiable) are always considered satisfied.
1317  */
1318 static int
1319 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1320 {
1321         graph_edge_t *edge;
1322         graph_vertex_t *v;
1323         boolean_t any_qualified;
1324         boolean_t any_unsatisfied;
1325         int i;
1326 
1327         any_qualified = B_FALSE;
1328         any_unsatisfied = B_FALSE;
1329 
1330         for (edge = uu_list_first(groupv->gv_dependencies);
1331             edge != NULL;
1332             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1333                 v = edge->ge_vertex;
1334 
1335                 switch (v->gv_type) {
1336                 case GVT_INST:
1337                         /* Skip missing instances */
1338                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1339                                 continue;
1340 
1341                         if (v->gv_state == RESTARTER_STATE_MAINT)
1342                                 continue;
1343 
1344                         any_qualified = B_TRUE;
1345                         if (v->gv_state == RESTARTER_STATE_OFFLINE ||
1346                             v->gv_state == RESTARTER_STATE_DISABLED) {
1347                                 /*
1348                                  * For offline/disabled dependencies,
1349                                  * treat unsatisfiable as satisfied.
1350                                  */
1351                                 i = dependency_satisfied(v, B_TRUE);
1352                                 if (i == -1)
1353                                         i = 1;
1354                         } else {
1355                                 i = dependency_satisfied(v, satbility);
1356                         }
1357                         break;
1358 
1359                 case GVT_FILE:
1360                         any_qualified = B_TRUE;
1361                         i = dependency_satisfied(v, satbility);
1362 
1363                         break;
1364 
1365                 case GVT_SVC: {
1366                         any_qualified = B_TRUE;
1367                         i = optional_all_satisfied(v, satbility);
1368 
1369                         break;
1370                 }
1371 
1372                 case GVT_GROUP:
1373                 default:
1374 #ifndef NDEBUG
1375                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1376                             __LINE__, v->gv_type);
1377 #endif
1378                         abort();
1379                 }
1380 
1381                 if (i == 1)
1382                         continue;
1383 
1384                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1385                     "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1386                     v->gv_name, i == 0 ? "ed" : "able");
1387 
1388                 if (!satbility)
1389                         return (0);
1390                 if (i == -1)
1391                         return (-1);
1392                 any_unsatisfied = B_TRUE;
1393         }
1394 
1395         if (!any_qualified)
1396                 return (1);
1397 
1398         return (any_unsatisfied ? 0 : 1);
1399 }
1400 
1401 /*
1402  * An exclude_all dependency is unsatisfied if any non-service element is
1403  * satisfied or any service instance which is configured, enabled, and not in
1404  * maintenance is satisfied.  Usually when unsatisfied, it is also
1405  * unsatisfiable.
1406  */
1407 #define LOG_EXCLUDE(u, v)                                               \
1408         log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,                   \
1409             "exclude_all(%s): %s is satisfied.\n",                      \
1410             (u)->gv_name, (v)->gv_name)
1411 
1412 /* ARGSUSED */
1413 static int
1414 exclude_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1415 {
1416         graph_edge_t *edge, *e2;
1417         graph_vertex_t *v, *v2;
1418 
1419         for (edge = uu_list_first(groupv->gv_dependencies);
1420             edge != NULL;
1421             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1422                 v = edge->ge_vertex;
1423 
1424                 switch (v->gv_type) {
1425                 case GVT_INST:
1426                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1427                                 continue;
1428 
1429                         switch (v->gv_state) {
1430                         case RESTARTER_STATE_ONLINE:
1431                         case RESTARTER_STATE_DEGRADED:
1432                                 LOG_EXCLUDE(groupv, v);
1433                                 return (v->gv_flags & GV_ENABLED ? -1 : 0);
1434 
1435                         case RESTARTER_STATE_OFFLINE:
1436                         case RESTARTER_STATE_UNINIT:
1437                                 LOG_EXCLUDE(groupv, v);
1438                                 return (0);
1439 
1440                         case RESTARTER_STATE_DISABLED:
1441                         case RESTARTER_STATE_MAINT:
1442                                 continue;
1443 
1444                         default:
1445 #ifndef NDEBUG
1446                                 uu_warn("%s:%d: Unexpected vertex state %d.\n",
1447                                     __FILE__, __LINE__, v->gv_state);
1448 #endif
1449                                 abort();
1450                         }
1451                         /* NOTREACHED */
1452 
1453                 case GVT_SVC:
1454                         break;
1455 
1456                 case GVT_FILE:
1457                         if (!file_ready(v))
1458                                 continue;
1459                         LOG_EXCLUDE(groupv, v);
1460                         return (-1);
1461 
1462                 case GVT_GROUP:
1463                 default:
1464 #ifndef NDEBUG
1465                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1466                             __LINE__, v->gv_type);
1467 #endif
1468                         abort();
1469                 }
1470 
1471                 /* v represents a service */
1472                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1473                         continue;
1474 
1475                 for (e2 = uu_list_first(v->gv_dependencies);
1476                     e2 != NULL;
1477                     e2 = uu_list_next(v->gv_dependencies, e2)) {
1478                         v2 = e2->ge_vertex;
1479                         assert(v2->gv_type == GVT_INST);
1480 
1481                         if ((v2->gv_flags & GV_CONFIGURED) == 0)
1482                                 continue;
1483 
1484                         switch (v2->gv_state) {
1485                         case RESTARTER_STATE_ONLINE:
1486                         case RESTARTER_STATE_DEGRADED:
1487                                 LOG_EXCLUDE(groupv, v2);
1488                                 return (v2->gv_flags & GV_ENABLED ? -1 : 0);
1489 
1490                         case RESTARTER_STATE_OFFLINE:
1491                         case RESTARTER_STATE_UNINIT:
1492                                 LOG_EXCLUDE(groupv, v2);
1493                                 return (0);
1494 
1495                         case RESTARTER_STATE_DISABLED:
1496                         case RESTARTER_STATE_MAINT:
1497                                 continue;
1498 
1499                         default:
1500 #ifndef NDEBUG
1501                                 uu_warn("%s:%d: Unexpected vertex type %d.\n",
1502                                     __FILE__, __LINE__, v2->gv_type);
1503 #endif
1504                                 abort();
1505                         }
1506                 }
1507         }
1508 
1509         return (1);
1510 }
1511 
1512 /*
1513  * int instance_satisfied()
1514  *   Determine if all the dependencies are satisfied for the supplied instance
1515  *   vertex. Return 1 if they are, 0 if they aren't, and -1 if they won't be
1516  *   without administrator intervention.
1517  */
1518 static int
1519 instance_satisfied(graph_vertex_t *v, boolean_t satbility)
1520 {
1521         assert(v->gv_type == GVT_INST);
1522         assert(!inst_running(v));
1523 
1524         return (require_all_satisfied(v, satbility));
1525 }
1526 
1527 /*
1528  * Decide whether v can satisfy a dependency.  v can either be a child of
1529  * a group vertex, or of an instance vertex.
1530  */
1531 static int
1532 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1533 {
1534         switch (v->gv_type) {
1535         case GVT_INST:
1536                 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1537                         if (v->gv_flags & GV_DEATHROW) {
1538                                 /*
1539                                  * A dependency on an instance with GV_DEATHROW
1540                                  * flag is always considered as satisfied.
1541                                  */
1542                                 return (1);
1543                         }
1544                         return (-1);
1545                 }
1546 
1547                 /*
1548                  * Vertices may be transitioning so we try to figure out if
1549                  * the end state is likely to satisfy the dependency instead
1550                  * of assuming the dependency is unsatisfied/unsatisfiable.
1551                  *
1552                  * Support for optional_all dependencies depends on us getting
1553                  * this right because unsatisfiable dependencies are treated
1554                  * as being satisfied.
1555                  */
1556                 switch (v->gv_state) {
1557                 case RESTARTER_STATE_ONLINE:
1558                 case RESTARTER_STATE_DEGRADED:
1559                         if (v->gv_flags & GV_TODISABLE)
1560                                 return (-1);
1561                         if (v->gv_flags & GV_TOOFFLINE)
1562                                 return (0);
1563                         return (1);
1564 
1565                 case RESTARTER_STATE_OFFLINE:
1566                         if (!satbility || v->gv_flags & GV_TODISABLE)
1567                                 return (satbility ? -1 : 0);
1568                         return (instance_satisfied(v, satbility) != -1 ?
1569                             0 : -1);
1570 
1571                 case RESTARTER_STATE_DISABLED:
1572                         if (!satbility || !(v->gv_flags & GV_ENABLED))
1573                                 return (satbility ? -1 : 0);
1574                         return (instance_satisfied(v, satbility) != -1 ?
1575                             0 : -1);
1576 
1577                 case RESTARTER_STATE_MAINT:
1578                         return (-1);
1579 
1580                 case RESTARTER_STATE_UNINIT:
1581                         return (0);
1582 
1583                 default:
1584 #ifndef NDEBUG
1585                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
1586                             __FILE__, __LINE__, v->gv_state);
1587 #endif
1588                         abort();
1589                         /* NOTREACHED */
1590                 }
1591 
1592         case GVT_SVC:
1593                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1594                         return (-1);
1595                 return (require_any_satisfied(v, satbility));
1596 
1597         case GVT_FILE:
1598                 /* i.e., we assume files will not be automatically generated */
1599                 return (file_ready(v) ? 1 : -1);
1600 
1601         case GVT_GROUP:
1602                 break;
1603 
1604         default:
1605 #ifndef NDEBUG
1606                 uu_warn("%s:%d: Unexpected node type %d.\n", __FILE__, __LINE__,
1607                     v->gv_type);
1608 #endif
1609                 abort();
1610                 /* NOTREACHED */
1611         }
1612 
1613         switch (v->gv_depgroup) {
1614         case DEPGRP_REQUIRE_ANY:
1615                 return (require_any_satisfied(v, satbility));
1616 
1617         case DEPGRP_REQUIRE_ALL:
1618                 return (require_all_satisfied(v, satbility));
1619 
1620         case DEPGRP_OPTIONAL_ALL:
1621                 return (optional_all_satisfied(v, satbility));
1622 
1623         case DEPGRP_EXCLUDE_ALL:
1624                 return (exclude_all_satisfied(v, satbility));
1625 
1626         default:
1627 #ifndef NDEBUG
1628                 uu_warn("%s:%d: Unknown dependency grouping %d.\n", __FILE__,
1629                     __LINE__, v->gv_depgroup);
1630 #endif
1631                 abort();
1632         }
1633 }
1634 
1635 void
1636 graph_start_if_satisfied(graph_vertex_t *v)
1637 {
1638         if (v->gv_state == RESTARTER_STATE_OFFLINE &&
1639             instance_satisfied(v, B_FALSE) == 1) {
1640                 if (v->gv_start_f == NULL)
1641                         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
1642                 else
1643                         v->gv_start_f(v);
1644         }
1645 }
1646 
1647 /*
1648  * propagate_satbility()
1649  *
1650  * This function is used when the given vertex changes state in such a way that
1651  * one of its dependents may become unsatisfiable.  This happens when an
1652  * instance transitions between offline -> online, or from !running ->
1653  * maintenance, as well as when an instance is removed from the graph.
1654  *
1655  * We have to walk all the dependents, since optional_all dependencies several
1656  * levels up could become (un)satisfied, instead of unsatisfiable.  For example,
1657  *
1658  *      +-----+  optional_all  +-----+  require_all  +-----+
1659  *      |  A  |--------------->|  B  |-------------->|  C  |
1660  *      +-----+                +-----+               +-----+
1661  *
1662  *                                              offline -> maintenance
1663  *
1664  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1665  * an optional dependency, what was previously an unsatisfiable situation is now
1666  * satisfied (B will never come online, even though its state hasn't changed).
1667  *
1668  * Note that it's not necessary to continue examining dependents after reaching
1669  * an optional_all dependency.  It's not possible for an optional_all dependency
1670  * to change satisfiability without also coming online, in which case we get a
1671  * start event and propagation continues naturally.  However, it does no harm to
1672  * continue propagating satisfiability (as it is a relatively rare event), and
1673  * keeps the walker code simple and generic.
1674  */
1675 /*ARGSUSED*/
1676 static int
1677 satbility_cb(graph_vertex_t *v, void *arg)
1678 {
1679         if (is_inst_bypassed(v))
1680                 return (UU_WALK_NEXT);
1681 
1682         if (v->gv_type == GVT_INST)
1683                 graph_start_if_satisfied(v);
1684 
1685         return (UU_WALK_NEXT);
1686 }
1687 
1688 static void
1689 propagate_satbility(graph_vertex_t *v)
1690 {
1691         graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1692 }
1693 
1694 static void propagate_stop(graph_vertex_t *, void *);
1695 
1696 /*
1697  * propagate_start()
1698  *
1699  * This function is used to propagate a start event to the dependents of the
1700  * given vertex.  Any dependents that are offline but have their dependencies
1701  * satisfied are started.  Any dependents that are online and have restart_on
1702  * set to "restart" or "refresh" are restarted because their dependencies have
1703  * just changed.  This only happens with optional_all dependencies.
1704  */
1705 static void
1706 propagate_start(graph_vertex_t *v, void *arg)
1707 {
1708         restarter_error_t err = (restarter_error_t)arg;
1709 
1710         if (is_inst_bypassed(v))
1711                 return;
1712 
1713         switch (v->gv_type) {
1714         case GVT_INST:
1715                 /* Restarter */
1716                 if (inst_running(v)) {
1717                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1718                                 vertex_send_event(v,
1719                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1720                         }
1721                 } else {
1722                         graph_start_if_satisfied(v);
1723                 }
1724                 break;
1725 
1726         case GVT_GROUP:
1727                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1728                         graph_walk_dependents(v, propagate_stop,
1729                             (void *)RERR_RESTART);
1730                         break;
1731                 }
1732                 err = v->gv_restart;
1733                 /* FALLTHROUGH */
1734 
1735         case GVT_SVC:
1736                 graph_walk_dependents(v, propagate_start, (void *)err);
1737                 break;
1738 
1739         case GVT_FILE:
1740 #ifndef NDEBUG
1741                 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1742                     __FILE__, __LINE__);
1743 #endif
1744                 abort();
1745                 /* NOTREACHED */
1746 
1747         default:
1748 #ifndef NDEBUG
1749                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1750                     v->gv_type);
1751 #endif
1752                 abort();
1753         }
1754 }
1755 
1756 /*
1757  * propagate_stop()
1758  *
1759  * This function is used to propagate a stop event to the dependents of the
1760  * given vertex.  Any dependents that are online (or in degraded state) with
1761  * the restart_on property set to "restart" or "refresh" will be stopped as
1762  * their dependencies have just changed, propagate_start() will start them
1763  * again once their dependencies have been re-satisfied.
1764  */
1765 static void
1766 propagate_stop(graph_vertex_t *v, void *arg)
1767 {
1768         restarter_error_t err = (restarter_error_t)arg;
1769 
1770         if (is_inst_bypassed(v))
1771                 return;
1772 
1773         switch (v->gv_type) {
1774         case GVT_INST:
1775                 /* Restarter */
1776                 if (err > RERR_NONE && inst_running(v)) {
1777                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1778                                 vertex_send_event(v,
1779                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1780                         } else {
1781                                 vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1782                         }
1783                 }
1784                 break;
1785 
1786         case GVT_SVC:
1787                 graph_walk_dependents(v, propagate_stop, arg);
1788                 break;
1789 
1790         case GVT_FILE:
1791 #ifndef NDEBUG
1792                 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1793                     __FILE__, __LINE__);
1794 #endif
1795                 abort();
1796                 /* NOTREACHED */
1797 
1798         case GVT_GROUP:
1799                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1800                         graph_walk_dependents(v, propagate_start,
1801                             (void *)RERR_NONE);
1802                         break;
1803                 }
1804 
1805                 if (err == RERR_NONE || err > v->gv_restart)
1806                         break;
1807 
1808                 graph_walk_dependents(v, propagate_stop, arg);
1809                 break;
1810 
1811         default:
1812 #ifndef NDEBUG
1813                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1814                     v->gv_type);
1815 #endif
1816                 abort();
1817         }
1818 }
1819 
1820 void
1821 offline_vertex(graph_vertex_t *v)
1822 {
1823         scf_handle_t *h = libscf_handle_create_bound_loop();
1824         scf_instance_t *scf_inst = safe_scf_instance_create(h);
1825         scf_propertygroup_t *pg = safe_scf_pg_create(h);
1826         restarter_instance_state_t state, next_state;
1827         int r;
1828 
1829         assert(v->gv_type == GVT_INST);
1830 
1831         if (scf_inst == NULL)
1832                 bad_error("safe_scf_instance_create", scf_error());
1833         if (pg == NULL)
1834                 bad_error("safe_scf_pg_create", scf_error());
1835 
1836         /* if the vertex is already going offline, return */
1837 rep_retry:
1838         if (scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, scf_inst, NULL,
1839             NULL, SCF_DECODE_FMRI_EXACT) != 0) {
1840                 switch (scf_error()) {
1841                 case SCF_ERROR_CONNECTION_BROKEN:
1842                         libscf_handle_rebind(h);
1843                         goto rep_retry;
1844 
1845                 case SCF_ERROR_NOT_FOUND:
1846                         scf_pg_destroy(pg);
1847                         scf_instance_destroy(scf_inst);
1848                         (void) scf_handle_unbind(h);
1849                         scf_handle_destroy(h);
1850                         return;
1851                 }
1852                 uu_die("Can't decode FMRI %s: %s\n", v->gv_name,
1853                     scf_strerror(scf_error()));
1854         }
1855 
1856         r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
1857         if (r != 0) {
1858                 switch (scf_error()) {
1859                 case SCF_ERROR_CONNECTION_BROKEN:
1860                         libscf_handle_rebind(h);
1861                         goto rep_retry;
1862 
1863                 case SCF_ERROR_NOT_SET:
1864                 case SCF_ERROR_NOT_FOUND:
1865                         scf_pg_destroy(pg);
1866                         scf_instance_destroy(scf_inst);
1867                         (void) scf_handle_unbind(h);
1868                         scf_handle_destroy(h);
1869                         return;
1870 
1871                 default:
1872                         bad_error("scf_instance_get_pg", scf_error());
1873                 }
1874         } else {
1875                 r = libscf_read_states(pg, &state, &next_state);
1876                 if (r == 0 && (next_state == RESTARTER_STATE_OFFLINE ||
1877                     next_state == RESTARTER_STATE_DISABLED)) {
1878                         log_framework(LOG_DEBUG,
1879                             "%s: instance is already going down.\n",
1880                             v->gv_name);
1881                         scf_pg_destroy(pg);
1882                         scf_instance_destroy(scf_inst);
1883                         (void) scf_handle_unbind(h);
1884                         scf_handle_destroy(h);
1885                         return;
1886                 }
1887         }
1888 
1889         scf_pg_destroy(pg);
1890         scf_instance_destroy(scf_inst);
1891         (void) scf_handle_unbind(h);
1892         scf_handle_destroy(h);
1893 
1894         vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP_RESET);
1895 }
1896 
1897 /*
1898  * void graph_enable_by_vertex()
1899  *   If admin is non-zero, this is an administrative request for change
1900  *   of the enabled property.  Thus, send the ADMIN_DISABLE rather than
1901  *   a plain DISABLE restarter event.
1902  */
1903 void
1904 graph_enable_by_vertex(graph_vertex_t *vertex, int enable, int admin)
1905 {
1906         graph_vertex_t *v;
1907         int r;
1908 
1909         assert(MUTEX_HELD(&dgraph_lock));
1910         assert((vertex->gv_flags & GV_CONFIGURED));
1911 
1912         vertex->gv_flags = (vertex->gv_flags & ~GV_ENABLED) |
1913             (enable ? GV_ENABLED : 0);
1914 
1915         if (enable) {
1916                 if (vertex->gv_state != RESTARTER_STATE_OFFLINE &&
1917                     vertex->gv_state != RESTARTER_STATE_DEGRADED &&
1918                     vertex->gv_state != RESTARTER_STATE_ONLINE) {
1919                         /*
1920                          * In case the vertex was notified to go down,
1921                          * but now can return online, clear the _TOOFFLINE
1922                          * and _TODISABLE flags.
1923                          */
1924                         vertex->gv_flags &= ~GV_TOOFFLINE;
1925                         vertex->gv_flags &= ~GV_TODISABLE;
1926 
1927                         vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ENABLE);
1928                 }
1929 
1930                 /*
1931                  * Wait for state update from restarter before sending _START or
1932                  * _STOP.
1933                  */
1934 
1935                 return;
1936         }
1937 
1938         if (vertex->gv_state == RESTARTER_STATE_DISABLED)
1939                 return;
1940 
1941         if (!admin) {
1942                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_DISABLE);
1943 
1944                 /*
1945                  * Wait for state update from restarter before sending _START or
1946                  * _STOP.
1947                  */
1948 
1949                 return;
1950         }
1951 
1952         /*
1953          * If it is a DISABLE event requested by the administrator then we are
1954          * offlining the dependents first.
1955          */
1956 
1957         /*
1958          * Set GV_TOOFFLINE for the services we are offlining. We cannot
1959          * clear the GV_TOOFFLINE bits from all the services because
1960          * other DISABLE events might be handled at the same time.
1961          */
1962         vertex->gv_flags |= GV_TOOFFLINE;
1963 
1964         /* remember which vertex to disable... */
1965         vertex->gv_flags |= GV_TODISABLE;
1966 
1967         log_framework(LOG_DEBUG, "Marking in-subtree vertices before "
1968             "disabling %s.\n", vertex->gv_name);
1969 
1970         /* set GV_TOOFFLINE for its dependents */
1971         r = uu_list_walk(vertex->gv_dependents, (uu_walk_fn_t *)mark_subtree,
1972             NULL, 0);
1973         assert(r == 0);
1974 
1975         /* disable the instance now if there is nothing else to offline */
1976         if (insubtree_dependents_down(vertex) == B_TRUE) {
1977                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
1978                 return;
1979         }
1980 
1981         /*
1982          * This loop is similar to the one used for the graph reversal shutdown
1983          * and could be improved in term of performance for the subtree reversal
1984          * disable case.
1985          */
1986         for (v = uu_list_first(dgraph); v != NULL;
1987             v = uu_list_next(dgraph, v)) {
1988                 /* skip the vertex we are disabling for now */
1989                 if (v == vertex)
1990                         continue;
1991 
1992                 if (v->gv_type != GVT_INST ||
1993                     (v->gv_flags & GV_CONFIGURED) == 0 ||
1994                     (v->gv_flags & GV_ENABLED) == 0 ||
1995                     (v->gv_flags & GV_TOOFFLINE) == 0)
1996                         continue;
1997 
1998                 if ((v->gv_state != RESTARTER_STATE_ONLINE) &&
1999                     (v->gv_state != RESTARTER_STATE_DEGRADED)) {
2000                         /* continue if there is nothing to offline */
2001                         continue;
2002                 }
2003 
2004                 /*
2005                  * Instances which are up need to come down before we're
2006                  * done, but we can only offline the leaves here. An
2007                  * instance is a leaf when all its dependents are down.
2008                  */
2009                 if (insubtree_dependents_down(v) == B_TRUE) {
2010                         log_framework(LOG_DEBUG, "Offlining in-subtree "
2011                             "instance %s for %s.\n",
2012                             v->gv_name, vertex->gv_name);
2013                         offline_vertex(v);
2014                 }
2015         }
2016 }
2017 
2018 static int configure_vertex(graph_vertex_t *, scf_instance_t *);
2019 
2020 /*
2021  * Set the restarter for v to fmri_arg.  That is, make sure a vertex for
2022  * fmri_arg exists, make v depend on it, and send _ADD_INSTANCE for v.  If
2023  * v is already configured and fmri_arg indicates the current restarter, do
2024  * nothing.  If v is configured and fmri_arg is a new restarter, delete v's
2025  * dependency on the restarter, send _REMOVE_INSTANCE for v, and set the new
2026  * restarter.  Returns 0 on success, EINVAL if the FMRI is invalid,
2027  * ECONNABORTED if the repository connection is broken, and ELOOP
2028  * if the dependency would create a cycle.  In the last case, *pathp will
2029  * point to a -1-terminated array of ids which compose the path from v to
2030  * restarter_fmri.
2031  */
2032 int
2033 graph_change_restarter(graph_vertex_t *v, const char *fmri_arg, scf_handle_t *h,
2034     int **pathp)
2035 {
2036         char *restarter_fmri = NULL;
2037         graph_vertex_t *rv;
2038         int err;
2039         int id;
2040 
2041         assert(MUTEX_HELD(&dgraph_lock));
2042 
2043         if (fmri_arg[0] != '\0') {
2044                 err = fmri_canonify(fmri_arg, &restarter_fmri, B_TRUE);
2045                 if (err != 0) {
2046                         assert(err == EINVAL);
2047                         return (err);
2048                 }
2049         }
2050 
2051         if (restarter_fmri == NULL ||
2052             strcmp(restarter_fmri, SCF_SERVICE_STARTD) == 0) {
2053                 if (v->gv_flags & GV_CONFIGURED) {
2054                         if (v->gv_restarter_id == -1) {
2055                                 if (restarter_fmri != NULL)
2056                                         startd_free(restarter_fmri,
2057                                             max_scf_fmri_size);
2058                                 return (0);
2059                         }
2060 
2061                         graph_unset_restarter(v);
2062                 }
2063 
2064                 /* Master restarter, nothing to do. */
2065                 v->gv_restarter_id = -1;
2066                 v->gv_restarter_channel = NULL;
2067                 vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2068                 return (0);
2069         }
2070 
2071         if (v->gv_flags & GV_CONFIGURED) {
2072                 id = dict_lookup_byname(restarter_fmri);
2073                 if (id != -1 && v->gv_restarter_id == id) {
2074                         startd_free(restarter_fmri, max_scf_fmri_size);
2075                         return (0);
2076                 }
2077 
2078                 graph_unset_restarter(v);
2079         }
2080 
2081         err = graph_insert_vertex_unconfigured(restarter_fmri, GVT_INST, 0,
2082             RERR_NONE, &rv);
2083         startd_free(restarter_fmri, max_scf_fmri_size);
2084         assert(err == 0 || err == EEXIST);
2085 
2086         if (rv->gv_delegate_initialized == 0) {
2087                 if ((rv->gv_delegate_channel = restarter_protocol_init_delegate(
2088                     rv->gv_name)) == NULL)
2089                         return (EINVAL);
2090                 rv->gv_delegate_initialized = 1;
2091         }
2092         v->gv_restarter_id = rv->gv_id;
2093         v->gv_restarter_channel = rv->gv_delegate_channel;
2094 
2095         err = graph_insert_dependency(v, rv, pathp);
2096         if (err != 0) {
2097                 assert(err == ELOOP);
2098                 return (ELOOP);
2099         }
2100 
2101         vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2102 
2103         if (!(rv->gv_flags & GV_CONFIGURED)) {
2104                 scf_instance_t *inst;
2105 
2106                 err = libscf_fmri_get_instance(h, rv->gv_name, &inst);
2107                 switch (err) {
2108                 case 0:
2109                         err = configure_vertex(rv, inst);
2110                         scf_instance_destroy(inst);
2111                         switch (err) {
2112                         case 0:
2113                         case ECANCELED:
2114                                 break;
2115 
2116                         case ECONNABORTED:
2117                                 return (ECONNABORTED);
2118 
2119                         default:
2120                                 bad_error("configure_vertex", err);
2121                         }
2122                         break;
2123 
2124                 case ECONNABORTED:
2125                         return (ECONNABORTED);
2126 
2127                 case ENOENT:
2128                         break;
2129 
2130                 case ENOTSUP:
2131                         /*
2132                          * The fmri doesn't specify an instance - translate
2133                          * to EINVAL.
2134                          */
2135                         return (EINVAL);
2136 
2137                 case EINVAL:
2138                 default:
2139                         bad_error("libscf_fmri_get_instance", err);
2140                 }
2141         }
2142 
2143         return (0);
2144 }
2145 
2146 
2147 /*
2148  * Add all of the instances of the service named by fmri to the graph.
2149  * Returns
2150  *   0 - success
2151  *   ENOENT - service indicated by fmri does not exist
2152  *
2153  * In both cases *reboundp will be B_TRUE if the handle was rebound, or B_FALSE
2154  * otherwise.
2155  */
2156 static int
2157 add_service(const char *fmri, scf_handle_t *h, boolean_t *reboundp)
2158 {
2159         scf_service_t *svc;
2160         scf_instance_t *inst;
2161         scf_iter_t *iter;
2162         char *inst_fmri;
2163         int ret, r;
2164 
2165         *reboundp = B_FALSE;
2166 
2167         svc = safe_scf_service_create(h);
2168         inst = safe_scf_instance_create(h);
2169         iter = safe_scf_iter_create(h);
2170         inst_fmri = startd_alloc(max_scf_fmri_size);
2171 
2172 rebound:
2173         if (scf_handle_decode_fmri(h, fmri, NULL, svc, NULL, NULL, NULL,
2174             SCF_DECODE_FMRI_EXACT) != 0) {
2175                 switch (scf_error()) {
2176                 case SCF_ERROR_CONNECTION_BROKEN:
2177                 default:
2178                         libscf_handle_rebind(h);
2179                         *reboundp = B_TRUE;
2180                         goto rebound;
2181 
2182                 case SCF_ERROR_NOT_FOUND:
2183                         ret = ENOENT;
2184                         goto out;
2185 
2186                 case SCF_ERROR_INVALID_ARGUMENT:
2187                 case SCF_ERROR_CONSTRAINT_VIOLATED:
2188                 case SCF_ERROR_NOT_BOUND:
2189                 case SCF_ERROR_HANDLE_MISMATCH:
2190                         bad_error("scf_handle_decode_fmri", scf_error());
2191                 }
2192         }
2193 
2194         if (scf_iter_service_instances(iter, svc) != 0) {
2195                 switch (scf_error()) {
2196                 case SCF_ERROR_CONNECTION_BROKEN:
2197                 default:
2198                         libscf_handle_rebind(h);
2199                         *reboundp = B_TRUE;
2200                         goto rebound;
2201 
2202                 case SCF_ERROR_DELETED:
2203                         ret = ENOENT;
2204                         goto out;
2205 
2206                 case SCF_ERROR_HANDLE_MISMATCH:
2207                 case SCF_ERROR_NOT_BOUND:
2208                 case SCF_ERROR_NOT_SET:
2209                         bad_error("scf_iter_service_instances", scf_error());
2210                 }
2211         }
2212 
2213         for (;;) {
2214                 r = scf_iter_next_instance(iter, inst);
2215                 if (r == 0)
2216                         break;
2217                 if (r != 1) {
2218                         switch (scf_error()) {
2219                         case SCF_ERROR_CONNECTION_BROKEN:
2220                         default:
2221                                 libscf_handle_rebind(h);
2222                                 *reboundp = B_TRUE;
2223                                 goto rebound;
2224 
2225                         case SCF_ERROR_DELETED:
2226                                 ret = ENOENT;
2227                                 goto out;
2228 
2229                         case SCF_ERROR_HANDLE_MISMATCH:
2230                         case SCF_ERROR_NOT_BOUND:
2231                         case SCF_ERROR_NOT_SET:
2232                         case SCF_ERROR_INVALID_ARGUMENT:
2233                                 bad_error("scf_iter_next_instance",
2234                                     scf_error());
2235                         }
2236                 }
2237 
2238                 if (scf_instance_to_fmri(inst, inst_fmri, max_scf_fmri_size) <
2239                     0) {
2240                         switch (scf_error()) {
2241                         case SCF_ERROR_CONNECTION_BROKEN:
2242                                 libscf_handle_rebind(h);
2243                                 *reboundp = B_TRUE;
2244                                 goto rebound;
2245 
2246                         case SCF_ERROR_DELETED:
2247                                 continue;
2248 
2249                         case SCF_ERROR_NOT_BOUND:
2250                         case SCF_ERROR_NOT_SET:
2251                                 bad_error("scf_instance_to_fmri", scf_error());
2252                         }
2253                 }
2254 
2255                 r = dgraph_add_instance(inst_fmri, inst, B_FALSE);
2256                 switch (r) {
2257                 case 0:
2258                 case ECANCELED:
2259                         break;
2260 
2261                 case EEXIST:
2262                         continue;
2263 
2264                 case ECONNABORTED:
2265                         libscf_handle_rebind(h);
2266                         *reboundp = B_TRUE;
2267                         goto rebound;
2268 
2269                 case EINVAL:
2270                 default:
2271                         bad_error("dgraph_add_instance", r);
2272                 }
2273         }
2274 
2275         ret = 0;
2276 
2277 out:
2278         startd_free(inst_fmri, max_scf_fmri_size);
2279         scf_iter_destroy(iter);
2280         scf_instance_destroy(inst);
2281         scf_service_destroy(svc);
2282         return (ret);
2283 }
2284 
2285 struct depfmri_info {
2286         graph_vertex_t  *v;             /* GVT_GROUP vertex */
2287         gv_type_t       type;           /* type of dependency */
2288         const char      *inst_fmri;     /* FMRI of parental GVT_INST vert. */
2289         const char      *pg_name;       /* Name of dependency pg */
2290         scf_handle_t    *h;
2291         int             err;            /* return error code */
2292         int             **pathp;        /* return circular dependency path */
2293 };
2294 
2295 /*
2296  * Find or create a vertex for fmri and make info->v depend on it.
2297  * Returns
2298  *   0 - success
2299  *   nonzero - failure
2300  *
2301  * On failure, sets info->err to
2302  *   EINVAL - fmri is invalid
2303  *            fmri does not match info->type
2304  *   ELOOP - Adding the dependency creates a circular dependency.  *info->pathp
2305  *           will point to an array of the ids of the members of the cycle.
2306  *   ECONNABORTED - repository connection was broken
2307  *   ECONNRESET - succeeded, but repository connection was reset
2308  */
2309 static int
2310 process_dependency_fmri(const char *fmri, struct depfmri_info *info)
2311 {
2312         int err;
2313         graph_vertex_t *depgroup_v, *v;
2314         char *fmri_copy, *cfmri;
2315         size_t fmri_copy_sz;
2316         const char *scope, *service, *instance, *pg;
2317         scf_instance_t *inst;
2318         boolean_t rebound;
2319 
2320         assert(MUTEX_HELD(&dgraph_lock));
2321 
2322         /* Get or create vertex for FMRI */
2323         depgroup_v = info->v;
2324 
2325         if (strncmp(fmri, "file:", sizeof ("file:") - 1) == 0) {
2326                 if (info->type != GVT_FILE) {
2327                         log_framework(LOG_NOTICE,
2328                             "FMRI \"%s\" is not allowed for the \"%s\" "
2329                             "dependency's type of instance %s.\n", fmri,
2330                             info->pg_name, info->inst_fmri);
2331                         return (info->err = EINVAL);
2332                 }
2333 
2334                 err = graph_insert_vertex_unconfigured(fmri, info->type, 0,
2335                     RERR_NONE, &v);
2336                 switch (err) {
2337                 case 0:
2338                         break;
2339 
2340                 case EEXIST:
2341                         assert(v->gv_type == GVT_FILE);
2342                         break;
2343 
2344                 case EINVAL:            /* prevented above */
2345                 default:
2346                         bad_error("graph_insert_vertex_unconfigured", err);
2347                 }
2348         } else {
2349                 if (info->type != GVT_INST) {
2350                         log_framework(LOG_NOTICE,
2351                             "FMRI \"%s\" is not allowed for the \"%s\" "
2352                             "dependency's type of instance %s.\n", fmri,
2353                             info->pg_name, info->inst_fmri);
2354                         return (info->err = EINVAL);
2355                 }
2356 
2357                 /*
2358                  * We must canonify fmri & add a vertex for it.
2359                  */
2360                 fmri_copy_sz = strlen(fmri) + 1;
2361                 fmri_copy = startd_alloc(fmri_copy_sz);
2362                 (void) strcpy(fmri_copy, fmri);
2363 
2364                 /* Determine if the FMRI is a property group or instance */
2365                 if (scf_parse_svc_fmri(fmri_copy, &scope, &service,
2366                     &instance, &pg, NULL) != 0) {
2367                         startd_free(fmri_copy, fmri_copy_sz);
2368                         log_framework(LOG_NOTICE,
2369                             "Dependency \"%s\" of %s has invalid FMRI "
2370                             "\"%s\".\n", info->pg_name, info->inst_fmri,
2371                             fmri);
2372                         return (info->err = EINVAL);
2373                 }
2374 
2375                 if (service == NULL || pg != NULL) {
2376                         startd_free(fmri_copy, fmri_copy_sz);
2377                         log_framework(LOG_NOTICE,
2378                             "Dependency \"%s\" of %s does not designate a "
2379                             "service or instance.\n", info->pg_name,
2380                             info->inst_fmri);
2381                         return (info->err = EINVAL);
2382                 }
2383 
2384                 if (scope == NULL || strcmp(scope, SCF_SCOPE_LOCAL) == 0) {
2385                         cfmri = uu_msprintf("svc:/%s%s%s",
2386                             service, instance ? ":" : "", instance ? instance :
2387                             "");
2388                 } else {
2389                         cfmri = uu_msprintf("svc://%s/%s%s%s",
2390                             scope, service, instance ? ":" : "", instance ?
2391                             instance : "");
2392                 }
2393 
2394                 startd_free(fmri_copy, fmri_copy_sz);
2395 
2396                 err = graph_insert_vertex_unconfigured(cfmri, instance ?
2397                     GVT_INST : GVT_SVC, instance ? 0 : DEPGRP_REQUIRE_ANY,
2398                     RERR_NONE, &v);
2399                 uu_free(cfmri);
2400                 switch (err) {
2401                 case 0:
2402                         break;
2403 
2404                 case EEXIST:
2405                         /* Verify v. */
2406                         if (instance != NULL)
2407                                 assert(v->gv_type == GVT_INST);
2408                         else
2409                                 assert(v->gv_type == GVT_SVC);
2410                         break;
2411 
2412                 default:
2413                         bad_error("graph_insert_vertex_unconfigured", err);
2414                 }
2415         }
2416 
2417         /* Add dependency from depgroup_v to new vertex */
2418         info->err = graph_insert_dependency(depgroup_v, v, info->pathp);
2419         switch (info->err) {
2420         case 0:
2421                 break;
2422 
2423         case ELOOP:
2424                 return (ELOOP);
2425 
2426         default:
2427                 bad_error("graph_insert_dependency", info->err);
2428         }
2429 
2430         /* This must be after we insert the dependency, to avoid looping. */
2431         switch (v->gv_type) {
2432         case GVT_INST:
2433                 if ((v->gv_flags & GV_CONFIGURED) != 0)
2434                         break;
2435 
2436                 inst = safe_scf_instance_create(info->h);
2437 
2438                 rebound = B_FALSE;
2439 
2440 rebound:
2441                 err = libscf_lookup_instance(v->gv_name, inst);
2442                 switch (err) {
2443                 case 0:
2444                         err = configure_vertex(v, inst);
2445                         switch (err) {
2446                         case 0:
2447                         case ECANCELED:
2448                                 break;
2449 
2450                         case ECONNABORTED:
2451                                 libscf_handle_rebind(info->h);
2452                                 rebound = B_TRUE;
2453                                 goto rebound;
2454 
2455                         default:
2456                                 bad_error("configure_vertex", err);
2457                         }
2458                         break;
2459 
2460                 case ENOENT:
2461                         break;
2462 
2463                 case ECONNABORTED:
2464                         libscf_handle_rebind(info->h);
2465                         rebound = B_TRUE;
2466                         goto rebound;
2467 
2468                 case EINVAL:
2469                 case ENOTSUP:
2470                 default:
2471                         bad_error("libscf_fmri_get_instance", err);
2472                 }
2473 
2474                 scf_instance_destroy(inst);
2475 
2476                 if (rebound)
2477                         return (info->err = ECONNRESET);
2478                 break;
2479 
2480         case GVT_SVC:
2481                 (void) add_service(v->gv_name, info->h, &rebound);
2482                 if (rebound)
2483                         return (info->err = ECONNRESET);
2484         }
2485 
2486         return (0);
2487 }
2488 
2489 struct deppg_info {
2490         graph_vertex_t  *v;             /* GVT_INST vertex */
2491         int             err;            /* return error */
2492         int             **pathp;        /* return circular dependency path */
2493 };
2494 
2495 /*
2496  * Make info->v depend on a new GVT_GROUP node for this property group,
2497  * and then call process_dependency_fmri() for the values of the entity
2498  * property.  Return 0 on success, or if something goes wrong return nonzero
2499  * and set info->err to ECONNABORTED, EINVAL, or the error code returned by
2500  * process_dependency_fmri().
2501  */
2502 static int
2503 process_dependency_pg(scf_propertygroup_t *pg, struct deppg_info *info)
2504 {
2505         scf_handle_t *h;
2506         depgroup_type_t deptype;
2507         restarter_error_t rerr;
2508         struct depfmri_info linfo;
2509         char *fmri, *pg_name;
2510         size_t fmri_sz;
2511         graph_vertex_t *depgrp;
2512         scf_property_t *prop;
2513         int err;
2514         int empty;
2515         scf_error_t scferr;
2516         ssize_t len;
2517 
2518         assert(MUTEX_HELD(&dgraph_lock));
2519 
2520         h = scf_pg_handle(pg);
2521 
2522         pg_name = startd_alloc(max_scf_name_size);
2523 
2524         len = scf_pg_get_name(pg, pg_name, max_scf_name_size);
2525         if (len < 0) {
2526                 startd_free(pg_name, max_scf_name_size);
2527                 switch (scf_error()) {
2528                 case SCF_ERROR_CONNECTION_BROKEN:
2529                 default:
2530                         return (info->err = ECONNABORTED);
2531 
2532                 case SCF_ERROR_DELETED:
2533                         return (info->err = 0);
2534 
2535                 case SCF_ERROR_NOT_SET:
2536                         bad_error("scf_pg_get_name", scf_error());
2537                 }
2538         }
2539 
2540         /*
2541          * Skip over empty dependency groups.  Since dependency property
2542          * groups are updated atomically, they are either empty or
2543          * fully populated.
2544          */
2545         empty = depgroup_empty(h, pg);
2546         if (empty < 0) {
2547                 log_error(LOG_INFO,
2548                     "Error reading dependency group \"%s\" of %s: %s\n",
2549                     pg_name, info->v->gv_name, scf_strerror(scf_error()));
2550                 startd_free(pg_name, max_scf_name_size);
2551                 return (info->err = EINVAL);
2552 
2553         } else if (empty == 1) {
2554                 log_framework(LOG_DEBUG,
2555                     "Ignoring empty dependency group \"%s\" of %s\n",
2556                     pg_name, info->v->gv_name);
2557                 startd_free(pg_name, max_scf_name_size);
2558                 return (info->err = 0);
2559         }
2560 
2561         fmri_sz = strlen(info->v->gv_name) + 1 + len + 1;
2562         fmri = startd_alloc(fmri_sz);
2563 
2564         (void) snprintf(fmri, fmri_sz, "%s>%s", info->v->gv_name,
2565             pg_name);
2566 
2567         /* Validate the pg before modifying the graph */
2568         deptype = depgroup_read_grouping(h, pg);
2569         if (deptype == DEPGRP_UNSUPPORTED) {
2570                 log_error(LOG_INFO,
2571                     "Dependency \"%s\" of %s has an unknown grouping value.\n",
2572                     pg_name, info->v->gv_name);
2573                 startd_free(fmri, fmri_sz);
2574                 startd_free(pg_name, max_scf_name_size);
2575                 return (info->err = EINVAL);
2576         }
2577 
2578         rerr = depgroup_read_restart(h, pg);
2579         if (rerr == RERR_UNSUPPORTED) {
2580                 log_error(LOG_INFO,
2581                     "Dependency \"%s\" of %s has an unknown restart_on value."
2582                     "\n", pg_name, info->v->gv_name);
2583                 startd_free(fmri, fmri_sz);
2584                 startd_free(pg_name, max_scf_name_size);
2585                 return (info->err = EINVAL);
2586         }
2587 
2588         prop = safe_scf_property_create(h);
2589 
2590         if (scf_pg_get_property(pg, SCF_PROPERTY_ENTITIES, prop) != 0) {
2591                 scferr = scf_error();
2592                 scf_property_destroy(prop);
2593                 if (scferr == SCF_ERROR_DELETED) {
2594                         startd_free(fmri, fmri_sz);
2595                         startd_free(pg_name, max_scf_name_size);
2596                         return (info->err = 0);
2597                 } else if (scferr != SCF_ERROR_NOT_FOUND) {
2598                         startd_free(fmri, fmri_sz);
2599                         startd_free(pg_name, max_scf_name_size);
2600                         return (info->err = ECONNABORTED);
2601                 }
2602 
2603                 log_error(LOG_INFO,
2604                     "Dependency \"%s\" of %s is missing a \"%s\" property.\n",
2605                     pg_name, info->v->gv_name, SCF_PROPERTY_ENTITIES);
2606 
2607                 startd_free(fmri, fmri_sz);
2608                 startd_free(pg_name, max_scf_name_size);
2609 
2610                 return (info->err = EINVAL);
2611         }
2612 
2613         /* Create depgroup vertex for pg */
2614         err = graph_insert_vertex_unconfigured(fmri, GVT_GROUP, deptype,
2615             rerr, &depgrp);
2616         assert(err == 0);
2617         startd_free(fmri, fmri_sz);
2618 
2619         /* Add dependency from inst vertex to new vertex */
2620         err = graph_insert_dependency(info->v, depgrp, info->pathp);
2621         /* ELOOP can't happen because this should be a new vertex */
2622         assert(err == 0);
2623 
2624         linfo.v = depgrp;
2625         linfo.type = depgroup_read_scheme(h, pg);
2626         linfo.inst_fmri = info->v->gv_name;
2627         linfo.pg_name = pg_name;
2628         linfo.h = h;
2629         linfo.err = 0;
2630         linfo.pathp = info->pathp;
2631         err = walk_property_astrings(prop, (callback_t)process_dependency_fmri,
2632             &linfo);
2633 
2634         scf_property_destroy(prop);
2635         startd_free(pg_name, max_scf_name_size);
2636 
2637         switch (err) {
2638         case 0:
2639         case EINTR:
2640                 return (info->err = linfo.err);
2641 
2642         case ECONNABORTED:
2643         case EINVAL:
2644                 return (info->err = err);
2645 
2646         case ECANCELED:
2647                 return (info->err = 0);
2648 
2649         case ECONNRESET:
2650                 return (info->err = ECONNABORTED);
2651 
2652         default:
2653                 bad_error("walk_property_astrings", err);
2654                 /* NOTREACHED */
2655         }
2656 }
2657 
2658 /*
2659  * Build the dependency info for v from the repository.  Returns 0 on success,
2660  * ECONNABORTED on repository disconnection, EINVAL if the repository
2661  * configuration is invalid, and ELOOP if a dependency would cause a cycle.
2662  * In the last case, *pathp will point to a -1-terminated array of ids which
2663  * constitute the rest of the dependency cycle.
2664  */
2665 static int
2666 set_dependencies(graph_vertex_t *v, scf_instance_t *inst, int **pathp)
2667 {
2668         struct deppg_info info;
2669         int err;
2670         uint_t old_configured;
2671 
2672         assert(MUTEX_HELD(&dgraph_lock));
2673 
2674         /*
2675          * Mark the vertex as configured during dependency insertion to avoid
2676          * dependency cycles (which can appear in the graph if one of the
2677          * vertices is an exclusion-group).
2678          */
2679         old_configured = v->gv_flags & GV_CONFIGURED;
2680         v->gv_flags |= GV_CONFIGURED;
2681 
2682         info.err = 0;
2683         info.v = v;
2684         info.pathp = pathp;
2685 
2686         err = walk_dependency_pgs(inst, (callback_t)process_dependency_pg,
2687             &info);
2688 
2689         if (!old_configured)
2690                 v->gv_flags &= ~GV_CONFIGURED;
2691 
2692         switch (err) {
2693         case 0:
2694         case EINTR:
2695                 return (info.err);
2696 
2697         case ECONNABORTED:
2698                 return (ECONNABORTED);
2699 
2700         case ECANCELED:
2701                 /* Should get delete event, so return 0. */
2702                 return (0);
2703 
2704         default:
2705                 bad_error("walk_dependency_pgs", err);
2706                 /* NOTREACHED */
2707         }
2708 }
2709 
2710 
2711 static void
2712 handle_cycle(const char *fmri, int *path)
2713 {
2714         const char *cp;
2715         size_t sz;
2716 
2717         assert(MUTEX_HELD(&dgraph_lock));
2718 
2719         path_to_str(path, (char **)&cp, &sz);
2720 
2721         log_error(LOG_ERR, "Transitioning %s to maintenance "
2722             "because it completes a dependency cycle (see svcs -xv for "
2723             "details):\n%s", fmri ? fmri : "?", cp);
2724 
2725         startd_free((void *)cp, sz);
2726 }
2727 
2728 /*
2729  * Increment the vertex's reference count to prevent the vertex removal
2730  * from the dgraph.
2731  */
2732 static void
2733 vertex_ref(graph_vertex_t *v)
2734 {
2735         assert(MUTEX_HELD(&dgraph_lock));
2736 
2737         v->gv_refs++;
2738 }
2739 
2740 /*
2741  * Decrement the vertex's reference count and remove the vertex from
2742  * the dgraph when possible.
2743  *
2744  * Return VERTEX_REMOVED when the vertex has been removed otherwise
2745  * return VERTEX_INUSE.
2746  */
2747 static int
2748 vertex_unref(graph_vertex_t *v)
2749 {
2750         assert(MUTEX_HELD(&dgraph_lock));
2751         assert(v->gv_refs > 0);
2752 
2753         v->gv_refs--;
2754 
2755         return (free_if_unrefed(v));
2756 }
2757 
2758 /*
2759  * When run on the dependencies of a vertex, populates list with
2760  * graph_edge_t's which point to the service vertices or the instance
2761  * vertices (no GVT_GROUP nodes) on which the vertex depends.
2762  *
2763  * Increment the vertex's reference count once the vertex is inserted
2764  * in the list. The vertex won't be able to be deleted from the dgraph
2765  * while it is referenced.
2766  */
2767 static int
2768 append_svcs_or_insts(graph_edge_t *e, uu_list_t *list)
2769 {
2770         graph_vertex_t *v = e->ge_vertex;
2771         graph_edge_t *new;
2772         int r;
2773 
2774         switch (v->gv_type) {
2775         case GVT_INST:
2776         case GVT_SVC:
2777                 break;
2778 
2779         case GVT_GROUP:
2780                 r = uu_list_walk(v->gv_dependencies,
2781                     (uu_walk_fn_t *)append_svcs_or_insts, list, 0);
2782                 assert(r == 0);
2783                 return (UU_WALK_NEXT);
2784 
2785         case GVT_FILE:
2786                 return (UU_WALK_NEXT);
2787 
2788         default:
2789 #ifndef NDEBUG
2790                 uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
2791                     __LINE__, v->gv_type);
2792 #endif
2793                 abort();
2794         }
2795 
2796         new = startd_alloc(sizeof (*new));
2797         new->ge_vertex = v;
2798         uu_list_node_init(new, &new->ge_link, graph_edge_pool);
2799         r = uu_list_insert_before(list, NULL, new);
2800         assert(r == 0);
2801 
2802         /*
2803          * Because we are inserting the vertex in a list, we don't want
2804          * the vertex to be freed while the list is in use. In order to
2805          * achieve that, increment the vertex's reference count.
2806          */
2807         vertex_ref(v);
2808 
2809         return (UU_WALK_NEXT);
2810 }
2811 
2812 static boolean_t
2813 should_be_in_subgraph(graph_vertex_t *v)
2814 {
2815         graph_edge_t *e;
2816 
2817         if (v == milestone)
2818                 return (B_TRUE);
2819 
2820         /*
2821          * v is in the subgraph if any of its dependents are in the subgraph.
2822          * Except for EXCLUDE_ALL dependents.  And OPTIONAL dependents only
2823          * count if we're enabled.
2824          */
2825         for (e = uu_list_first(v->gv_dependents);
2826             e != NULL;
2827             e = uu_list_next(v->gv_dependents, e)) {
2828                 graph_vertex_t *dv = e->ge_vertex;
2829 
2830                 if (!(dv->gv_flags & GV_INSUBGRAPH))
2831                         continue;
2832 
2833                 /*
2834                  * Don't include instances that are optional and disabled.
2835                  */
2836                 if (v->gv_type == GVT_INST && dv->gv_type == GVT_SVC) {
2837 
2838                         int in = 0;
2839                         graph_edge_t *ee;
2840 
2841                         for (ee = uu_list_first(dv->gv_dependents);
2842                             ee != NULL;
2843                             ee = uu_list_next(dv->gv_dependents, ee)) {
2844 
2845                                 graph_vertex_t *ddv = e->ge_vertex;
2846 
2847                                 if (ddv->gv_type == GVT_GROUP &&
2848                                     ddv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2849                                         continue;
2850 
2851                                 if (ddv->gv_type == GVT_GROUP &&
2852                                     ddv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2853                                     !(v->gv_flags & GV_ENBLD_NOOVR))
2854                                         continue;
2855 
2856                                 in = 1;
2857                         }
2858                         if (!in)
2859                                 continue;
2860                 }
2861                 if (v->gv_type == GVT_INST &&
2862                     dv->gv_type == GVT_GROUP &&
2863                     dv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2864                     !(v->gv_flags & GV_ENBLD_NOOVR))
2865                         continue;
2866 
2867                 /* Don't include excluded services and instances */
2868                 if (dv->gv_type == GVT_GROUP &&
2869                     dv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2870                         continue;
2871 
2872                 return (B_TRUE);
2873         }
2874 
2875         return (B_FALSE);
2876 }
2877 
2878 /*
2879  * Ensures that GV_INSUBGRAPH is set properly for v and its descendents.  If
2880  * any bits change, manipulate the repository appropriately.  Returns 0 or
2881  * ECONNABORTED.
2882  */
2883 static int
2884 eval_subgraph(graph_vertex_t *v, scf_handle_t *h)
2885 {
2886         boolean_t old = (v->gv_flags & GV_INSUBGRAPH) != 0;
2887         boolean_t new;
2888         graph_edge_t *e;
2889         scf_instance_t *inst;
2890         int ret = 0, r;
2891 
2892         assert(milestone != NULL && milestone != MILESTONE_NONE);
2893 
2894         new = should_be_in_subgraph(v);
2895 
2896         if (new == old)
2897                 return (0);
2898 
2899         log_framework(LOG_DEBUG, new ? "Adding %s to the subgraph.\n" :
2900             "Removing %s from the subgraph.\n", v->gv_name);
2901 
2902         v->gv_flags = (v->gv_flags & ~GV_INSUBGRAPH) |
2903             (new ? GV_INSUBGRAPH : 0);
2904 
2905         if (v->gv_type == GVT_INST && (v->gv_flags & GV_CONFIGURED)) {
2906                 int err;
2907 
2908 get_inst:
2909                 err = libscf_fmri_get_instance(h, v->gv_name, &inst);
2910                 if (err != 0) {
2911                         switch (err) {
2912                         case ECONNABORTED:
2913                                 libscf_handle_rebind(h);
2914                                 ret = ECONNABORTED;
2915                                 goto get_inst;
2916 
2917                         case ENOENT:
2918                                 break;
2919 
2920                         case EINVAL:
2921                         case ENOTSUP:
2922                         default:
2923                                 bad_error("libscf_fmri_get_instance", err);
2924                         }
2925                 } else {
2926                         const char *f;
2927 
2928                         if (new) {
2929                                 err = libscf_delete_enable_ovr(inst);
2930                                 f = "libscf_delete_enable_ovr";
2931                         } else {
2932                                 err = libscf_set_enable_ovr(inst, 0);
2933                                 f = "libscf_set_enable_ovr";
2934                         }
2935                         scf_instance_destroy(inst);
2936                         switch (err) {
2937                         case 0:
2938                         case ECANCELED:
2939                                 break;
2940 
2941                         case ECONNABORTED:
2942                                 libscf_handle_rebind(h);
2943                                 /*
2944                                  * We must continue so the graph is updated,
2945                                  * but we must return ECONNABORTED so any
2946                                  * libscf state held by any callers is reset.
2947                                  */
2948                                 ret = ECONNABORTED;
2949                                 goto get_inst;
2950 
2951                         case EROFS:
2952                         case EPERM:
2953                                 log_error(LOG_WARNING,
2954                                     "Could not set %s/%s for %s: %s.\n",
2955                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
2956                                     v->gv_name, strerror(err));
2957                                 break;
2958 
2959                         default:
2960                                 bad_error(f, err);
2961                         }
2962                 }
2963         }
2964 
2965         for (e = uu_list_first(v->gv_dependencies);
2966             e != NULL;
2967             e = uu_list_next(v->gv_dependencies, e)) {
2968                 r = eval_subgraph(e->ge_vertex, h);
2969                 if (r != 0) {
2970                         assert(r == ECONNABORTED);
2971                         ret = ECONNABORTED;
2972                 }
2973         }
2974 
2975         return (ret);
2976 }
2977 
2978 /*
2979  * Delete the (property group) dependencies of v & create new ones based on
2980  * inst.  If doing so would create a cycle, log a message and put the instance
2981  * into maintenance.  Update GV_INSUBGRAPH flags as necessary.  Returns 0 or
2982  * ECONNABORTED.
2983  */
2984 int
2985 refresh_vertex(graph_vertex_t *v, scf_instance_t *inst)
2986 {
2987         int err;
2988         int *path;
2989         char *fmri;
2990         int r;
2991         scf_handle_t *h = scf_instance_handle(inst);
2992         uu_list_t *old_deps;
2993         int ret = 0;
2994         graph_edge_t *e;
2995         graph_vertex_t *vv;
2996 
2997         assert(MUTEX_HELD(&dgraph_lock));
2998         assert(v->gv_type == GVT_INST);
2999 
3000         log_framework(LOG_DEBUG, "Graph engine: Refreshing %s.\n", v->gv_name);
3001 
3002         if (milestone > MILESTONE_NONE) {
3003                 /*
3004                  * In case some of v's dependencies are being deleted we must
3005                  * make a list of them now for GV_INSUBGRAPH-flag evaluation
3006                  * after the new dependencies are in place.
3007                  */
3008                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
3009 
3010                 err = uu_list_walk(v->gv_dependencies,
3011                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
3012                 assert(err == 0);
3013         }
3014 
3015         delete_instance_dependencies(v, B_FALSE);
3016 
3017         err = set_dependencies(v, inst, &path);
3018         switch (err) {
3019         case 0:
3020                 break;
3021 
3022         case ECONNABORTED:
3023                 ret = err;
3024                 goto out;
3025 
3026         case EINVAL:
3027         case ELOOP:
3028                 r = libscf_instance_get_fmri(inst, &fmri);
3029                 switch (r) {
3030                 case 0:
3031                         break;
3032 
3033                 case ECONNABORTED:
3034                         ret = ECONNABORTED;
3035                         goto out;
3036 
3037                 case ECANCELED:
3038                         ret = 0;
3039                         goto out;
3040 
3041                 default:
3042                         bad_error("libscf_instance_get_fmri", r);
3043                 }
3044 
3045                 if (err == EINVAL) {
3046                         log_error(LOG_ERR, "Transitioning %s "
3047                             "to maintenance due to misconfiguration.\n",
3048                             fmri ? fmri : "?");
3049                         vertex_send_event(v,
3050                             RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY);
3051                 } else {
3052                         handle_cycle(fmri, path);
3053                         vertex_send_event(v,
3054                             RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE);
3055                 }
3056                 startd_free(fmri, max_scf_fmri_size);
3057                 ret = 0;
3058                 goto out;
3059 
3060         default:
3061                 bad_error("set_dependencies", err);
3062         }
3063 
3064         if (milestone > MILESTONE_NONE) {
3065                 boolean_t aborted = B_FALSE;
3066 
3067                 for (e = uu_list_first(old_deps);
3068                     e != NULL;
3069                     e = uu_list_next(old_deps, e)) {
3070                         vv = e->ge_vertex;
3071 
3072                         if (vertex_unref(vv) == VERTEX_INUSE &&
3073                             eval_subgraph(vv, h) == ECONNABORTED)
3074                                 aborted = B_TRUE;
3075                 }
3076 
3077                 for (e = uu_list_first(v->gv_dependencies);
3078                     e != NULL;
3079                     e = uu_list_next(v->gv_dependencies, e)) {
3080                         if (eval_subgraph(e->ge_vertex, h) ==
3081                             ECONNABORTED)
3082                                 aborted = B_TRUE;
3083                 }
3084 
3085                 if (aborted) {
3086                         ret = ECONNABORTED;
3087                         goto out;
3088                 }
3089         }
3090 
3091         graph_start_if_satisfied(v);
3092 
3093         ret = 0;
3094 
3095 out:
3096         if (milestone > MILESTONE_NONE) {
3097                 void *cookie = NULL;
3098 
3099                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL)
3100                         startd_free(e, sizeof (*e));
3101 
3102                 uu_list_destroy(old_deps);
3103         }
3104 
3105         return (ret);
3106 }
3107 
3108 /*
3109  * Set up v according to inst.  That is, make sure it depends on its
3110  * restarter and set up its dependencies.  Send the ADD_INSTANCE command to
3111  * the restarter, and send ENABLE or DISABLE as appropriate.
3112  *
3113  * Returns 0 on success, ECONNABORTED on repository disconnection, or
3114  * ECANCELED if inst is deleted.
3115  */
3116 static int
3117 configure_vertex(graph_vertex_t *v, scf_instance_t *inst)
3118 {
3119         scf_handle_t *h;
3120         scf_propertygroup_t *pg;
3121         scf_snapshot_t *snap;
3122         char *restarter_fmri = startd_alloc(max_scf_value_size);
3123         int enabled, enabled_ovr;
3124         int err;
3125         int *path;
3126         int deathrow;
3127         int32_t tset;
3128 
3129         restarter_fmri[0] = '\0';
3130 
3131         assert(MUTEX_HELD(&dgraph_lock));
3132         assert(v->gv_type == GVT_INST);
3133         assert((v->gv_flags & GV_CONFIGURED) == 0);
3134 
3135         /* GV_INSUBGRAPH should already be set properly. */
3136         assert(should_be_in_subgraph(v) ==
3137             ((v->gv_flags & GV_INSUBGRAPH) != 0));
3138 
3139         /*
3140          * If the instance fmri is in the deathrow list then set the
3141          * GV_DEATHROW flag on the vertex and create and set to true the
3142          * SCF_PROPERTY_DEATHROW boolean property in the non-persistent
3143          * repository for this instance fmri.
3144          */
3145         if ((v->gv_flags & GV_DEATHROW) ||
3146             (is_fmri_in_deathrow(v->gv_name) == B_TRUE)) {
3147                 if ((v->gv_flags & GV_DEATHROW) == 0) {
3148                         /*
3149                          * Set flag GV_DEATHROW, create and set to true
3150                          * the SCF_PROPERTY_DEATHROW property in the
3151                          * non-persistent repository for this instance fmri.
3152                          */
3153                         v->gv_flags |= GV_DEATHROW;
3154 
3155                         switch (err = libscf_set_deathrow(inst, 1)) {
3156                         case 0:
3157                                 break;
3158 
3159                         case ECONNABORTED:
3160                         case ECANCELED:
3161                                 startd_free(restarter_fmri, max_scf_value_size);
3162                                 return (err);
3163 
3164                         case EROFS:
3165                                 log_error(LOG_WARNING, "Could not set %s/%s "
3166                                     "for deathrow %s: %s.\n",
3167                                     SCF_PG_DEATHROW, SCF_PROPERTY_DEATHROW,
3168                                     v->gv_name, strerror(err));
3169                                 break;
3170 
3171                         case EPERM:
3172                                 uu_die("Permission denied.\n");
3173                                 /* NOTREACHED */
3174 
3175                         default:
3176                                 bad_error("libscf_set_deathrow", err);
3177                         }
3178                         log_framework(LOG_DEBUG, "Deathrow, graph set %s.\n",
3179                             v->gv_name);
3180                 }
3181                 startd_free(restarter_fmri, max_scf_value_size);
3182                 return (0);
3183         }
3184 
3185         h = scf_instance_handle(inst);
3186 
3187         /*
3188          * Using a temporary deathrow boolean property, set through
3189          * libscf_set_deathrow(), only for fmris on deathrow, is necessary
3190          * because deathrow_fini() may already have been called, and in case
3191          * of a refresh, GV_DEATHROW may need to be set again.
3192          * libscf_get_deathrow() sets deathrow to 1 only if this instance
3193          * has a temporary boolean property named 'deathrow' valued true
3194          * in a property group 'deathrow', -1 or 0 in all other cases.
3195          */
3196         err = libscf_get_deathrow(h, inst, &deathrow);
3197         switch (err) {
3198         case 0:
3199                 break;
3200 
3201         case ECONNABORTED:
3202         case ECANCELED:
3203                 startd_free(restarter_fmri, max_scf_value_size);
3204                 return (err);
3205 
3206         default:
3207                 bad_error("libscf_get_deathrow", err);
3208         }
3209 
3210         if (deathrow == 1) {
3211                 v->gv_flags |= GV_DEATHROW;
3212                 startd_free(restarter_fmri, max_scf_value_size);
3213                 return (0);
3214         }
3215 
3216         log_framework(LOG_DEBUG, "Graph adding %s.\n", v->gv_name);
3217 
3218         /*
3219          * If the instance does not have a restarter property group,
3220          * initialize its state to uninitialized/none, in case the restarter
3221          * is not enabled.
3222          */
3223         pg = safe_scf_pg_create(h);
3224 
3225         if (scf_instance_get_pg(inst, SCF_PG_RESTARTER, pg) != 0) {
3226                 instance_data_t idata;
3227                 uint_t count = 0, msecs = ALLOC_DELAY;
3228 
3229                 switch (scf_error()) {
3230                 case SCF_ERROR_NOT_FOUND:
3231                         break;
3232 
3233                 case SCF_ERROR_CONNECTION_BROKEN:
3234                 default:
3235                         scf_pg_destroy(pg);
3236                         startd_free(restarter_fmri, max_scf_value_size);
3237                         return (ECONNABORTED);
3238 
3239                 case SCF_ERROR_DELETED:
3240                         scf_pg_destroy(pg);
3241                         startd_free(restarter_fmri, max_scf_value_size);
3242                         return (ECANCELED);
3243 
3244                 case SCF_ERROR_NOT_SET:
3245                         bad_error("scf_instance_get_pg", scf_error());
3246                 }
3247 
3248                 switch (err = libscf_instance_get_fmri(inst,
3249                     (char **)&idata.i_fmri)) {
3250                 case 0:
3251                         break;
3252 
3253                 case ECONNABORTED:
3254                 case ECANCELED:
3255                         scf_pg_destroy(pg);
3256                         startd_free(restarter_fmri, max_scf_value_size);
3257                         return (err);
3258 
3259                 default:
3260                         bad_error("libscf_instance_get_fmri", err);
3261                 }
3262 
3263                 idata.i_state = RESTARTER_STATE_NONE;
3264                 idata.i_next_state = RESTARTER_STATE_NONE;
3265 
3266 init_state:
3267                 switch (err = _restarter_commit_states(h, &idata,
3268                     RESTARTER_STATE_UNINIT, RESTARTER_STATE_NONE,
3269                     restarter_get_str_short(restarter_str_insert_in_graph))) {
3270                 case 0:
3271                         break;
3272 
3273                 case ENOMEM:
3274                         ++count;
3275                         if (count < ALLOC_RETRY) {
3276                                 (void) poll(NULL, 0, msecs);
3277                                 msecs *= ALLOC_DELAY_MULT;
3278                                 goto init_state;
3279                         }
3280 
3281                         uu_die("Insufficient memory.\n");
3282                         /* NOTREACHED */
3283 
3284                 case ECONNABORTED:
3285                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3286                         scf_pg_destroy(pg);
3287                         startd_free(restarter_fmri, max_scf_value_size);
3288                         return (ECONNABORTED);
3289 
3290                 case ENOENT:
3291                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3292                         scf_pg_destroy(pg);
3293                         startd_free(restarter_fmri, max_scf_value_size);
3294                         return (ECANCELED);
3295 
3296                 case EPERM:
3297                 case EACCES:
3298                 case EROFS:
3299                         log_error(LOG_NOTICE, "Could not initialize state for "
3300                             "%s: %s.\n", idata.i_fmri, strerror(err));
3301                         break;
3302 
3303                 case EINVAL:
3304                 default:
3305                         bad_error("_restarter_commit_states", err);
3306                 }
3307 
3308                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3309         }
3310 
3311         scf_pg_destroy(pg);
3312 
3313         if (milestone != NULL) {
3314                 /*
3315                  * Make sure the enable-override is set properly before we
3316                  * read whether we should be enabled.
3317                  */
3318                 if (milestone == MILESTONE_NONE ||
3319                     !(v->gv_flags & GV_INSUBGRAPH)) {
3320                         /*
3321                          * This might seem unjustified after the milestone
3322                          * transition has completed (non_subgraph_svcs == 0),
3323                          * but it's important because when we boot to
3324                          * a milestone, we set the milestone before populating
3325                          * the graph, and all of the new non-subgraph services
3326                          * need to be disabled here.
3327                          */
3328                         switch (err = libscf_set_enable_ovr(inst, 0)) {
3329                         case 0:
3330                                 break;
3331 
3332                         case ECONNABORTED:
3333                         case ECANCELED:
3334                                 startd_free(restarter_fmri, max_scf_value_size);
3335                                 return (err);
3336 
3337                         case EROFS:
3338                                 log_error(LOG_WARNING,
3339                                     "Could not set %s/%s for %s: %s.\n",
3340                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
3341                                     v->gv_name, strerror(err));
3342                                 break;
3343 
3344                         case EPERM:
3345                                 uu_die("Permission denied.\n");
3346                                 /* NOTREACHED */
3347 
3348                         default:
3349                                 bad_error("libscf_set_enable_ovr", err);
3350                         }
3351                 } else {
3352                         assert(v->gv_flags & GV_INSUBGRAPH);
3353                         switch (err = libscf_delete_enable_ovr(inst)) {
3354                         case 0:
3355                                 break;
3356 
3357                         case ECONNABORTED:
3358                         case ECANCELED:
3359                                 startd_free(restarter_fmri, max_scf_value_size);
3360                                 return (err);
3361 
3362                         case EPERM:
3363                                 uu_die("Permission denied.\n");
3364                                 /* NOTREACHED */
3365 
3366                         default:
3367                                 bad_error("libscf_delete_enable_ovr", err);
3368                         }
3369                 }
3370         }
3371 
3372         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
3373             &enabled_ovr, &restarter_fmri);
3374         switch (err) {
3375         case 0:
3376                 break;
3377 
3378         case ECONNABORTED:
3379         case ECANCELED:
3380                 startd_free(restarter_fmri, max_scf_value_size);
3381                 return (err);
3382 
3383         case ENOENT:
3384                 log_framework(LOG_DEBUG,
3385                     "Ignoring %s because it has no general property group.\n",
3386                     v->gv_name);
3387                 startd_free(restarter_fmri, max_scf_value_size);
3388                 return (0);
3389 
3390         default:
3391                 bad_error("libscf_get_basic_instance_data", err);
3392         }
3393 
3394         if ((tset = libscf_get_stn_tset(inst)) == -1) {
3395                 log_framework(LOG_WARNING,
3396                     "Failed to get notification parameters for %s: %s\n",
3397                     v->gv_name, scf_strerror(scf_error()));
3398                 v->gv_stn_tset = 0;
3399         } else {
3400                 v->gv_stn_tset = tset;
3401         }
3402         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
3403                 stn_global = v->gv_stn_tset;
3404 
3405         if (enabled == -1) {
3406                 startd_free(restarter_fmri, max_scf_value_size);
3407                 return (0);
3408         }
3409 
3410         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
3411             (enabled ? GV_ENBLD_NOOVR : 0);
3412 
3413         if (enabled_ovr != -1)
3414                 enabled = enabled_ovr;
3415 
3416         v->gv_state = RESTARTER_STATE_UNINIT;
3417 
3418         snap = libscf_get_or_make_running_snapshot(inst, v->gv_name, B_TRUE);
3419         scf_snapshot_destroy(snap);
3420 
3421         /* Set up the restarter. (Sends _ADD_INSTANCE on success.) */
3422         err = graph_change_restarter(v, restarter_fmri, h, &path);
3423         if (err != 0) {
3424                 instance_data_t idata;
3425                 uint_t count = 0, msecs = ALLOC_DELAY;
3426                 restarter_str_t reason;
3427 
3428                 if (err == ECONNABORTED) {
3429                         startd_free(restarter_fmri, max_scf_value_size);
3430                         return (err);
3431                 }
3432 
3433                 assert(err == EINVAL || err == ELOOP);
3434 
3435                 if (err == EINVAL) {
3436                         log_framework(LOG_ERR, emsg_invalid_restarter,
3437                             v->gv_name, restarter_fmri);
3438                         reason = restarter_str_invalid_restarter;
3439                 } else {
3440                         handle_cycle(v->gv_name, path);
3441                         reason = restarter_str_dependency_cycle;
3442                 }
3443 
3444                 startd_free(restarter_fmri, max_scf_value_size);
3445 
3446                 /*
3447                  * We didn't register the instance with the restarter, so we
3448                  * must set maintenance mode ourselves.
3449                  */
3450                 err = libscf_instance_get_fmri(inst, (char **)&idata.i_fmri);
3451                 if (err != 0) {
3452                         assert(err == ECONNABORTED || err == ECANCELED);
3453                         return (err);
3454                 }
3455 
3456                 idata.i_state = RESTARTER_STATE_NONE;
3457                 idata.i_next_state = RESTARTER_STATE_NONE;
3458 
3459 set_maint:
3460                 switch (err = _restarter_commit_states(h, &idata,
3461                     RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE,
3462                     restarter_get_str_short(reason))) {
3463                 case 0:
3464                         break;
3465 
3466                 case ENOMEM:
3467                         ++count;
3468                         if (count < ALLOC_RETRY) {
3469                                 (void) poll(NULL, 0, msecs);
3470                                 msecs *= ALLOC_DELAY_MULT;
3471                                 goto set_maint;
3472                         }
3473 
3474                         uu_die("Insufficient memory.\n");
3475                         /* NOTREACHED */
3476 
3477                 case ECONNABORTED:
3478                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3479                         return (ECONNABORTED);
3480 
3481                 case ENOENT:
3482                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3483                         return (ECANCELED);
3484 
3485                 case EPERM:
3486                 case EACCES:
3487                 case EROFS:
3488                         log_error(LOG_NOTICE, "Could not initialize state for "
3489                             "%s: %s.\n", idata.i_fmri, strerror(err));
3490                         break;
3491 
3492                 case EINVAL:
3493                 default:
3494                         bad_error("_restarter_commit_states", err);
3495                 }
3496 
3497                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3498 
3499                 v->gv_state = RESTARTER_STATE_MAINT;
3500 
3501                 goto out;
3502         }
3503         startd_free(restarter_fmri, max_scf_value_size);
3504 
3505         /* Add all the other dependencies. */
3506         err = refresh_vertex(v, inst);
3507         if (err != 0) {
3508                 assert(err == ECONNABORTED);
3509                 return (err);
3510         }
3511 
3512 out:
3513         v->gv_flags |= GV_CONFIGURED;
3514 
3515         graph_enable_by_vertex(v, enabled, 0);
3516 
3517         return (0);
3518 }
3519 
3520 
3521 static void
3522 kill_user_procs(void)
3523 {
3524         (void) fputs("svc.startd: Killing user processes.\n", stdout);
3525 
3526         /*
3527          * Despite its name, killall's role is to get select user processes--
3528          * basically those representing terminal-based logins-- to die.  Victims
3529          * are located by killall in the utmp database.  Since these are most
3530          * often shell based logins, and many shells mask SIGTERM (but are
3531          * responsive to SIGHUP) we first HUP and then shortly thereafter
3532          * kill -9.
3533          */
3534         (void) fork_with_timeout("/usr/sbin/killall HUP", 1, 5);
3535         (void) fork_with_timeout("/usr/sbin/killall KILL", 1, 5);
3536 
3537         /*
3538          * Note the selection of user id's 0, 1 and 15, subsequently
3539          * inverted by -v.  15 is reserved for dladmd.  Yes, this is a
3540          * kludge-- a better policy is needed.
3541          *
3542          * Note that fork_with_timeout will only wait out the 1 second
3543          * "grace time" if pkill actually returns 0.  So if there are
3544          * no matches, this will run to completion much more quickly.
3545          */
3546         (void) fork_with_timeout("/usr/bin/pkill -TERM -v -u 0,1,15", 1, 5);
3547         (void) fork_with_timeout("/usr/bin/pkill -KILL -v -u 0,1,15", 1, 5);
3548 }
3549 
3550 static void
3551 do_uadmin(void)
3552 {
3553         const char * const resetting = "/etc/svc/volatile/resetting";
3554         int fd;
3555         struct statvfs vfs;
3556         time_t now;
3557         struct tm nowtm;
3558         char down_buf[256], time_buf[256];
3559         uintptr_t mdep;
3560 #if defined(__x86)
3561         char *fbarg = NULL;
3562 #endif  /* __x86 */
3563 
3564         mdep = NULL;
3565         fd = creat(resetting, 0777);
3566         if (fd >= 0)
3567                 startd_close(fd);
3568         else
3569                 uu_warn("Could not create \"%s\"", resetting);
3570 
3571         /* Kill dhcpagent if we're not using nfs for root */
3572         if ((statvfs("/", &vfs) == 0) &&
3573             (strncmp(vfs.f_basetype, "nfs", sizeof ("nfs") - 1) != 0))
3574                 fork_with_timeout("/usr/bin/pkill -x -u 0 dhcpagent", 0, 5);
3575 
3576         /*
3577          * Call sync(2) now, before we kill off user processes.  This takes
3578          * advantage of the several seconds of pause we have before the
3579          * killalls are done.  Time we can make good use of to get pages
3580          * moving out to disk.
3581          *
3582          * Inside non-global zones, we don't bother, and it's better not to
3583          * anyway, since sync(2) can have system-wide impact.
3584          */
3585         if (getzoneid() == 0)
3586                 sync();
3587 
3588         kill_user_procs();
3589 
3590         /*
3591          * Note that this must come after the killing of user procs, since
3592          * killall relies on utmpx, and this command affects the contents of
3593          * said file.
3594          */
3595         if (access("/usr/lib/acct/closewtmp", X_OK) == 0)
3596                 fork_with_timeout("/usr/lib/acct/closewtmp", 0, 5);
3597 
3598         /*
3599          * For patches which may be installed as the system is shutting
3600          * down, we need to ensure, one more time, that the boot archive
3601          * really is up to date.
3602          */
3603         if (getzoneid() == 0 && access("/usr/sbin/bootadm", X_OK) == 0)
3604                 fork_with_timeout("/usr/sbin/bootadm -ea update_all", 0, 3600);
3605 
3606         /*
3607          * Right now, fast reboot is supported only on i386.
3608          * scf_is_fastboot_default() should take care of it.
3609          * If somehow we got there on unsupported platform -
3610          * print warning and fall back to regular reboot.
3611          */
3612         if (halting == AD_FASTREBOOT) {
3613 #if defined(__x86)
3614                 if (be_get_boot_args(&fbarg, BE_ENTRY_DEFAULT) == 0) {
3615                         mdep = (uintptr_t)fbarg;
3616                 } else {
3617                         /*
3618                          * Failed to read BE info, fall back to normal reboot
3619                          */
3620                         halting = AD_BOOT;
3621                         uu_warn("Failed to get fast reboot arguments.\n"
3622                             "Falling back to regular reboot.\n");
3623                 }
3624 #else   /* __x86 */
3625                 halting = AD_BOOT;
3626                 uu_warn("Fast reboot configured, but not supported by "
3627                     "this ISA\n");
3628 #endif  /* __x86 */
3629         }
3630 
3631         fork_with_timeout("/sbin/umountall -l", 0, 5);
3632         fork_with_timeout("/sbin/umount /tmp /var/adm /var/run /var "
3633             ">/dev/null 2>&1", 0, 5);
3634 
3635         /*
3636          * Try to get to consistency for whatever UFS filesystems are left.
3637          * This is pretty expensive, so we save it for the end in the hopes of
3638          * minimizing what it must do.  The other option would be to start in
3639          * parallel with the killall's, but lockfs tends to throw out much more
3640          * than is needed, and so subsequent commands (like umountall) take a
3641          * long time to get going again.
3642          *
3643          * Inside of zones, we don't bother, since we're not about to terminate
3644          * the whole OS instance.
3645          *
3646          * On systems using only ZFS, this call to lockfs -fa is a no-op.
3647          */
3648         if (getzoneid() == 0) {
3649                 if (access("/usr/sbin/lockfs", X_OK) == 0)
3650                         fork_with_timeout("/usr/sbin/lockfs -fa", 0, 30);
3651 
3652                 sync(); /* once more, with feeling */
3653         }
3654 
3655         fork_with_timeout("/sbin/umount /usr >/dev/null 2>&1", 0, 5);
3656 
3657         /*
3658          * Construct and emit the last words from userland:
3659          * "<timestamp> The system is down.  Shutdown took <N> seconds."
3660          *
3661          * Normally we'd use syslog, but with /var and other things
3662          * potentially gone, try to minimize the external dependencies.
3663          */
3664         now = time(NULL);
3665         (void) localtime_r(&now, &nowtm);
3666 
3667         if (strftime(down_buf, sizeof (down_buf),
3668             "%b %e %T The system is down.", &nowtm) == 0) {
3669                 (void) strlcpy(down_buf, "The system is down.",
3670                     sizeof (down_buf));
3671         }
3672 
3673         if (halting_time != 0 && halting_time <= now) {
3674                 (void) snprintf(time_buf, sizeof (time_buf),
3675                     "  Shutdown took %lu seconds.", now - halting_time);
3676         } else {
3677                 time_buf[0] = '\0';
3678         }
3679         (void) printf("%s%s\n", down_buf, time_buf);
3680 
3681         (void) uadmin(A_SHUTDOWN, halting, mdep);
3682         uu_warn("uadmin() failed");
3683 
3684 #if defined(__x86)
3685         if (halting == AD_FASTREBOOT)
3686                 free(fbarg);
3687 #endif  /* __x86 */
3688 
3689         if (remove(resetting) != 0 && errno != ENOENT)
3690                 uu_warn("Could not remove \"%s\"", resetting);
3691 }
3692 
3693 /*
3694  * If any of the up_svcs[] are online or satisfiable, return true.  If they are
3695  * all missing, disabled, in maintenance, or unsatisfiable, return false.
3696  */
3697 boolean_t
3698 can_come_up(void)
3699 {
3700         int i;
3701 
3702         assert(MUTEX_HELD(&dgraph_lock));
3703 
3704         /*
3705          * If we are booting to single user (boot -s),
3706          * SCF_MILESTONE_SINGLE_USER is needed to come up because startd
3707          * spawns sulogin after single-user is online (see specials.c).
3708          */
3709         i = (booting_to_single_user ? 0 : 1);
3710 
3711         for (; up_svcs[i] != NULL; ++i) {
3712                 if (up_svcs_p[i] == NULL) {
3713                         up_svcs_p[i] = vertex_get_by_name(up_svcs[i]);
3714 
3715                         if (up_svcs_p[i] == NULL)
3716                                 continue;
3717                 }
3718 
3719                 /*
3720                  * Ignore unconfigured services (the ones that have been
3721                  * mentioned in a dependency from other services, but do
3722                  * not exist in the repository).  Services which exist
3723                  * in the repository but don't have general/enabled
3724                  * property will be also ignored.
3725                  */
3726                 if (!(up_svcs_p[i]->gv_flags & GV_CONFIGURED))
3727                         continue;
3728 
3729                 switch (up_svcs_p[i]->gv_state) {
3730                 case RESTARTER_STATE_ONLINE:
3731                 case RESTARTER_STATE_DEGRADED:
3732                         /*
3733                          * Deactivate verbose boot once a login service has been
3734                          * reached.
3735                          */
3736                         st->st_log_login_reached = 1;
3737                         /*FALLTHROUGH*/
3738                 case RESTARTER_STATE_UNINIT:
3739                         return (B_TRUE);
3740 
3741                 case RESTARTER_STATE_OFFLINE:
3742                         if (instance_satisfied(up_svcs_p[i], B_TRUE) != -1)
3743                                 return (B_TRUE);
3744                         log_framework(LOG_DEBUG,
3745                             "can_come_up(): %s is unsatisfiable.\n",
3746                             up_svcs_p[i]->gv_name);
3747                         continue;
3748 
3749                 case RESTARTER_STATE_DISABLED:
3750                 case RESTARTER_STATE_MAINT:
3751                         log_framework(LOG_DEBUG,
3752                             "can_come_up(): %s is in state %s.\n",
3753                             up_svcs_p[i]->gv_name,
3754                             instance_state_str[up_svcs_p[i]->gv_state]);
3755                         continue;
3756 
3757                 default:
3758 #ifndef NDEBUG
3759                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
3760                             __FILE__, __LINE__, up_svcs_p[i]->gv_state);
3761 #endif
3762                         abort();
3763                 }
3764         }
3765 
3766         /*
3767          * In the seed repository, console-login is unsatisfiable because
3768          * services are missing.  To behave correctly in that case we don't want
3769          * to return false until manifest-import is online.
3770          */
3771 
3772         if (manifest_import_p == NULL) {
3773                 manifest_import_p = vertex_get_by_name(manifest_import);
3774 
3775                 if (manifest_import_p == NULL)
3776                         return (B_FALSE);
3777         }
3778 
3779         switch (manifest_import_p->gv_state) {
3780         case RESTARTER_STATE_ONLINE:
3781         case RESTARTER_STATE_DEGRADED:
3782         case RESTARTER_STATE_DISABLED:
3783         case RESTARTER_STATE_MAINT:
3784                 break;
3785 
3786         case RESTARTER_STATE_OFFLINE:
3787                 if (instance_satisfied(manifest_import_p, B_TRUE) == -1)
3788                         break;
3789                 /* FALLTHROUGH */
3790 
3791         case RESTARTER_STATE_UNINIT:
3792                 return (B_TRUE);
3793         }
3794 
3795         return (B_FALSE);
3796 }
3797 
3798 /*
3799  * Runs sulogin.  Returns
3800  *   0 - success
3801  *   EALREADY - sulogin is already running
3802  *   EBUSY - console-login is running
3803  */
3804 static int
3805 run_sulogin(const char *msg)
3806 {
3807         graph_vertex_t *v;
3808 
3809         assert(MUTEX_HELD(&dgraph_lock));
3810 
3811         if (sulogin_running)
3812                 return (EALREADY);
3813 
3814         v = vertex_get_by_name(console_login_fmri);
3815         if (v != NULL && inst_running(v))
3816                 return (EBUSY);
3817 
3818         sulogin_running = B_TRUE;
3819 
3820         MUTEX_UNLOCK(&dgraph_lock);
3821 
3822         fork_sulogin(B_FALSE, msg);
3823 
3824         MUTEX_LOCK(&dgraph_lock);
3825 
3826         sulogin_running = B_FALSE;
3827 
3828         if (console_login_ready) {
3829                 v = vertex_get_by_name(console_login_fmri);
3830 
3831                 if (v != NULL && v->gv_state == RESTARTER_STATE_OFFLINE) {
3832                         if (v->gv_start_f == NULL)
3833                                 vertex_send_event(v,
3834                                     RESTARTER_EVENT_TYPE_START);
3835                         else
3836                                 v->gv_start_f(v);
3837                 }
3838 
3839                 console_login_ready = B_FALSE;
3840         }
3841 
3842         return (0);
3843 }
3844 
3845 /*
3846  * The sulogin thread runs sulogin while can_come_up() is false.  run_sulogin()
3847  * keeps sulogin from stepping on console-login's toes.
3848  */
3849 /* ARGSUSED */
3850 static void *
3851 sulogin_thread(void *unused)
3852 {
3853         (void) pthread_setname_np(pthread_self(), "sulogin");
3854 
3855         MUTEX_LOCK(&dgraph_lock);
3856 
3857         assert(sulogin_thread_running);
3858 
3859         do {
3860                 (void) run_sulogin("Console login service(s) cannot run\n");
3861         } while (!can_come_up());
3862 
3863         sulogin_thread_running = B_FALSE;
3864         MUTEX_UNLOCK(&dgraph_lock);
3865 
3866         return (NULL);
3867 }
3868 
3869 /* ARGSUSED */
3870 void *
3871 single_user_thread(void *unused)
3872 {
3873         uint_t left;
3874         scf_handle_t *h;
3875         scf_instance_t *inst;
3876         scf_property_t *prop;
3877         scf_value_t *val;
3878         const char *msg;
3879         char *buf;
3880         int r;
3881 
3882         (void) pthread_setname_np(pthread_self(), "single_user");
3883 
3884         MUTEX_LOCK(&single_user_thread_lock);
3885         single_user_thread_count++;
3886 
3887         if (!booting_to_single_user)
3888                 kill_user_procs();
3889 
3890         if (go_single_user_mode || booting_to_single_user) {
3891                 msg = "SINGLE USER MODE\n";
3892         } else {
3893                 assert(go_to_level1);
3894 
3895                 fork_rc_script('1', "start", B_TRUE);
3896 
3897                 uu_warn("The system is ready for administration.\n");
3898 
3899                 msg = "";
3900         }
3901 
3902         MUTEX_UNLOCK(&single_user_thread_lock);
3903 
3904         for (;;) {
3905                 MUTEX_LOCK(&dgraph_lock);
3906                 r = run_sulogin(msg);
3907                 MUTEX_UNLOCK(&dgraph_lock);
3908                 if (r == 0)
3909                         break;
3910 
3911                 assert(r == EALREADY || r == EBUSY);
3912 
3913                 left = 3;
3914                 while (left > 0)
3915                         left = sleep(left);
3916         }
3917 
3918         MUTEX_LOCK(&single_user_thread_lock);
3919 
3920         /*
3921          * If another single user thread has started, let it finish changing
3922          * the run level.
3923          */
3924         if (single_user_thread_count > 1) {
3925                 single_user_thread_count--;
3926                 MUTEX_UNLOCK(&single_user_thread_lock);
3927                 return (NULL);
3928         }
3929 
3930         h = libscf_handle_create_bound_loop();
3931         inst = scf_instance_create(h);
3932         prop = safe_scf_property_create(h);
3933         val = safe_scf_value_create(h);
3934         buf = startd_alloc(max_scf_fmri_size);
3935 
3936 lookup:
3937         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
3938             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
3939                 switch (scf_error()) {
3940                 case SCF_ERROR_NOT_FOUND:
3941                         r = libscf_create_self(h);
3942                         if (r == 0)
3943                                 goto lookup;
3944                         assert(r == ECONNABORTED);
3945                         /* FALLTHROUGH */
3946 
3947                 case SCF_ERROR_CONNECTION_BROKEN:
3948                         libscf_handle_rebind(h);
3949                         goto lookup;
3950 
3951                 case SCF_ERROR_INVALID_ARGUMENT:
3952                 case SCF_ERROR_CONSTRAINT_VIOLATED:
3953                 case SCF_ERROR_NOT_BOUND:
3954                 case SCF_ERROR_HANDLE_MISMATCH:
3955                 default:
3956                         bad_error("scf_handle_decode_fmri", scf_error());
3957                 }
3958         }
3959 
3960         MUTEX_LOCK(&dgraph_lock);
3961 
3962         r = scf_instance_delete_prop(inst, SCF_PG_OPTIONS_OVR,
3963             SCF_PROPERTY_MILESTONE);
3964         switch (r) {
3965         case 0:
3966         case ECANCELED:
3967                 break;
3968 
3969         case ECONNABORTED:
3970                 MUTEX_UNLOCK(&dgraph_lock);
3971                 libscf_handle_rebind(h);
3972                 goto lookup;
3973 
3974         case EPERM:
3975         case EACCES:
3976         case EROFS:
3977                 log_error(LOG_WARNING, "Could not clear temporary milestone: "
3978                     "%s.\n", strerror(r));
3979                 break;
3980 
3981         default:
3982                 bad_error("scf_instance_delete_prop", r);
3983         }
3984 
3985         MUTEX_UNLOCK(&dgraph_lock);
3986 
3987         r = libscf_get_milestone(inst, prop, val, buf, max_scf_fmri_size);
3988         switch (r) {
3989         case ECANCELED:
3990         case ENOENT:
3991         case EINVAL:
3992                 (void) strcpy(buf, "all");
3993                 /* FALLTHROUGH */
3994 
3995         case 0:
3996                 uu_warn("Returning to milestone %s.\n", buf);
3997                 break;
3998 
3999         case ECONNABORTED:
4000                 libscf_handle_rebind(h);
4001                 goto lookup;
4002 
4003         default:
4004                 bad_error("libscf_get_milestone", r);
4005         }
4006 
4007         r = dgraph_set_milestone(buf, h, B_FALSE);
4008         switch (r) {
4009         case 0:
4010         case ECONNRESET:
4011         case EALREADY:
4012         case EINVAL:
4013         case ENOENT:
4014                 break;
4015 
4016         default:
4017                 bad_error("dgraph_set_milestone", r);
4018         }
4019 
4020         /*
4021          * See graph_runlevel_changed().
4022          */
4023         MUTEX_LOCK(&dgraph_lock);
4024         utmpx_set_runlevel(target_milestone_as_runlevel(), 'S', B_TRUE);
4025         MUTEX_UNLOCK(&dgraph_lock);
4026 
4027         startd_free(buf, max_scf_fmri_size);
4028         scf_value_destroy(val);
4029         scf_property_destroy(prop);
4030         scf_instance_destroy(inst);
4031         scf_handle_destroy(h);
4032 
4033         /*
4034          * We'll give ourselves 3 seconds to respond to all of the enablings
4035          * that setting the milestone should have created before checking
4036          * whether to run sulogin.
4037          */
4038         left = 3;
4039         while (left > 0)
4040                 left = sleep(left);
4041 
4042         MUTEX_LOCK(&dgraph_lock);
4043         /*
4044          * Clearing these variables will allow the sulogin thread to run.  We
4045          * check here in case there aren't any more state updates anytime soon.
4046          */
4047         go_to_level1 = go_single_user_mode = booting_to_single_user = B_FALSE;
4048         if (!sulogin_thread_running && !can_come_up()) {
4049                 (void) startd_thread_create(sulogin_thread, NULL);
4050                 sulogin_thread_running = B_TRUE;
4051         }
4052         MUTEX_UNLOCK(&dgraph_lock);
4053         single_user_thread_count--;
4054         MUTEX_UNLOCK(&single_user_thread_lock);
4055         return (NULL);
4056 }
4057 
4058 
4059 /*
4060  * Dependency graph operations API.  These are handle-independent thread-safe
4061  * graph manipulation functions which are the entry points for the event
4062  * threads below.
4063  */
4064 
4065 /*
4066  * If a configured vertex exists for inst_fmri, return EEXIST.  If no vertex
4067  * exists for inst_fmri, add one.  Then fetch the restarter from inst, make
4068  * this vertex dependent on it, and send _ADD_INSTANCE to the restarter.
4069  * Fetch whether the instance should be enabled from inst and send _ENABLE or
4070  * _DISABLE as appropriate.  Finally rummage through inst's dependency
4071  * property groups and add vertices and edges as appropriate.  If anything
4072  * goes wrong after sending _ADD_INSTANCE, send _ADMIN_MAINT_ON to put the
4073  * instance in maintenance.  Don't send _START or _STOP until we get a state
4074  * update in case we're being restarted and the service is already running.
4075  *
4076  * To support booting to a milestone, we must also make sure all dependencies
4077  * encountered are configured, if they exist in the repository.
4078  *
4079  * Returns 0 on success, ECONNABORTED on repository disconnection, EINVAL if
4080  * inst_fmri is an invalid (or not canonical) FMRI, ECANCELED if inst is
4081  * deleted, or EEXIST if a configured vertex for inst_fmri already exists.
4082  */
4083 int
4084 dgraph_add_instance(const char *inst_fmri, scf_instance_t *inst,
4085     boolean_t lock_graph)
4086 {
4087         graph_vertex_t *v;
4088         int err;
4089 
4090         if (strcmp(inst_fmri, SCF_SERVICE_STARTD) == 0)
4091                 return (0);
4092 
4093         /* Check for a vertex for inst_fmri. */
4094         if (lock_graph) {
4095                 MUTEX_LOCK(&dgraph_lock);
4096         } else {
4097                 assert(MUTEX_HELD(&dgraph_lock));
4098         }
4099 
4100         v = vertex_get_by_name(inst_fmri);
4101 
4102         if (v != NULL) {
4103                 assert(v->gv_type == GVT_INST);
4104 
4105                 if (v->gv_flags & GV_CONFIGURED) {
4106                         if (lock_graph)
4107                                 MUTEX_UNLOCK(&dgraph_lock);
4108                         return (EEXIST);
4109                 }
4110         } else {
4111                 /* Add the vertex. */
4112                 err = graph_insert_vertex_unconfigured(inst_fmri, GVT_INST, 0,
4113                     RERR_NONE, &v);
4114                 if (err != 0) {
4115                         assert(err == EINVAL);
4116                         if (lock_graph)
4117                                 MUTEX_UNLOCK(&dgraph_lock);
4118                         return (EINVAL);
4119                 }
4120         }
4121 
4122         err = configure_vertex(v, inst);
4123 
4124         if (lock_graph)
4125                 MUTEX_UNLOCK(&dgraph_lock);
4126 
4127         return (err);
4128 }
4129 
4130 /*
4131  * Locate the vertex for this property group's instance.  If it doesn't exist
4132  * or is unconfigured, call dgraph_add_instance() & return.  Otherwise fetch
4133  * the restarter for the instance, and if it has changed, send
4134  * _REMOVE_INSTANCE to the old restarter, remove the dependency, make sure the
4135  * new restarter has a vertex, add a new dependency, and send _ADD_INSTANCE to
4136  * the new restarter.  Then fetch whether the instance should be enabled, and
4137  * if it is different from what we had, or if we changed the restarter, send
4138  * the appropriate _ENABLE or _DISABLE command.
4139  *
4140  * Returns 0 on success, ENOTSUP if the pg's parent is not an instance,
4141  * ECONNABORTED on repository disconnection, ECANCELED if the instance is
4142  * deleted, or -1 if the instance's general property group is deleted or if
4143  * its enabled property is misconfigured.
4144  */
4145 static int
4146 dgraph_update_general(scf_propertygroup_t *pg)
4147 {
4148         scf_handle_t *h;
4149         scf_instance_t *inst;
4150         char *fmri;
4151         char *restarter_fmri;
4152         graph_vertex_t *v;
4153         int err;
4154         int enabled, enabled_ovr;
4155         int oldflags;
4156 
4157         /* Find the vertex for this service */
4158         h = scf_pg_handle(pg);
4159 
4160         inst = safe_scf_instance_create(h);
4161 
4162         if (scf_pg_get_parent_instance(pg, inst) != 0) {
4163                 switch (scf_error()) {
4164                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4165                         return (ENOTSUP);
4166 
4167                 case SCF_ERROR_CONNECTION_BROKEN:
4168                 default:
4169                         return (ECONNABORTED);
4170 
4171                 case SCF_ERROR_DELETED:
4172                         return (0);
4173 
4174                 case SCF_ERROR_NOT_SET:
4175                         bad_error("scf_pg_get_parent_instance", scf_error());
4176                 }
4177         }
4178 
4179         err = libscf_instance_get_fmri(inst, &fmri);
4180         switch (err) {
4181         case 0:
4182                 break;
4183 
4184         case ECONNABORTED:
4185                 scf_instance_destroy(inst);
4186                 return (ECONNABORTED);
4187 
4188         case ECANCELED:
4189                 scf_instance_destroy(inst);
4190                 return (0);
4191 
4192         default:
4193                 bad_error("libscf_instance_get_fmri", err);
4194         }
4195 
4196         log_framework(LOG_DEBUG,
4197             "Graph engine: Reloading general properties for %s.\n", fmri);
4198 
4199         MUTEX_LOCK(&dgraph_lock);
4200 
4201         v = vertex_get_by_name(fmri);
4202         if (v == NULL || !(v->gv_flags & GV_CONFIGURED)) {
4203                 /* Will get the up-to-date properties. */
4204                 MUTEX_UNLOCK(&dgraph_lock);
4205                 err = dgraph_add_instance(fmri, inst, B_TRUE);
4206                 startd_free(fmri, max_scf_fmri_size);
4207                 scf_instance_destroy(inst);
4208                 return (err == ECANCELED ? 0 : err);
4209         }
4210 
4211         /* Read enabled & restarter from repository. */
4212         restarter_fmri = startd_alloc(max_scf_value_size);
4213         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
4214             &enabled_ovr, &restarter_fmri);
4215         if (err != 0 || enabled == -1) {
4216                 MUTEX_UNLOCK(&dgraph_lock);
4217                 scf_instance_destroy(inst);
4218                 startd_free(fmri, max_scf_fmri_size);
4219 
4220                 switch (err) {
4221                 case ENOENT:
4222                 case 0:
4223                         startd_free(restarter_fmri, max_scf_value_size);
4224                         return (-1);
4225 
4226                 case ECONNABORTED:
4227                 case ECANCELED:
4228                         startd_free(restarter_fmri, max_scf_value_size);
4229                         return (err);
4230 
4231                 default:
4232                         bad_error("libscf_get_basic_instance_data", err);
4233                 }
4234         }
4235 
4236         oldflags = v->gv_flags;
4237         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
4238             (enabled ? GV_ENBLD_NOOVR : 0);
4239 
4240         if (enabled_ovr != -1)
4241                 enabled = enabled_ovr;
4242 
4243         /*
4244          * If GV_ENBLD_NOOVR has changed, then we need to re-evaluate the
4245          * subgraph.
4246          */
4247         if (milestone > MILESTONE_NONE && v->gv_flags != oldflags)
4248                 (void) eval_subgraph(v, h);
4249 
4250         scf_instance_destroy(inst);
4251 
4252         /* Ignore restarter change for now. */
4253 
4254         startd_free(restarter_fmri, max_scf_value_size);
4255         startd_free(fmri, max_scf_fmri_size);
4256 
4257         /*
4258          * Always send _ENABLE or _DISABLE.  We could avoid this if the
4259          * restarter didn't change and the enabled value didn't change, but
4260          * that's not easy to check and improbable anyway, so we'll just do
4261          * this.
4262          */
4263         graph_enable_by_vertex(v, enabled, 1);
4264 
4265         MUTEX_UNLOCK(&dgraph_lock);
4266 
4267         return (0);
4268 }
4269 
4270 /*
4271  * Delete all of the property group dependencies of v, update inst's running
4272  * snapshot, and add the dependencies in the new snapshot.  If any of the new
4273  * dependencies would create a cycle, send _ADMIN_MAINT_ON.  Otherwise
4274  * reevaluate v's dependencies, send _START or _STOP as appropriate, and do
4275  * the same for v's dependents.
4276  *
4277  * Returns
4278  *   0 - success
4279  *   ECONNABORTED - repository connection broken
4280  *   ECANCELED - inst was deleted
4281  *   EINVAL - inst is invalid (e.g., missing general/enabled)
4282  *   -1 - libscf_snapshots_refresh() failed
4283  */
4284 static int
4285 dgraph_refresh_instance(graph_vertex_t *v, scf_instance_t *inst)
4286 {
4287         int r;
4288         int enabled;
4289         int32_t tset;
4290 
4291         assert(MUTEX_HELD(&dgraph_lock));
4292         assert(v->gv_type == GVT_INST);
4293 
4294         /* Only refresh services with valid general/enabled properties. */
4295         r = libscf_get_basic_instance_data(scf_instance_handle(inst), inst,
4296             v->gv_name, &enabled, NULL, NULL);
4297         switch (r) {
4298         case 0:
4299                 break;
4300 
4301         case ECONNABORTED:
4302         case ECANCELED:
4303                 return (r);
4304 
4305         case ENOENT:
4306                 log_framework(LOG_DEBUG,
4307                     "Ignoring %s because it has no general property group.\n",
4308                     v->gv_name);
4309                 return (EINVAL);
4310 
4311         default:
4312                 bad_error("libscf_get_basic_instance_data", r);
4313         }
4314 
4315         if ((tset = libscf_get_stn_tset(inst)) == -1) {
4316                 log_framework(LOG_WARNING,
4317                     "Failed to get notification parameters for %s: %s\n",
4318                     v->gv_name, scf_strerror(scf_error()));
4319                 tset = 0;
4320         }
4321         v->gv_stn_tset = tset;
4322         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
4323                 stn_global = tset;
4324 
4325         if (enabled == -1)
4326                 return (EINVAL);
4327 
4328         r = libscf_snapshots_refresh(inst, v->gv_name);
4329         if (r != 0) {
4330                 if (r != -1)
4331                         bad_error("libscf_snapshots_refresh", r);
4332 
4333                 /* error logged */
4334                 return (r);
4335         }
4336 
4337         r = refresh_vertex(v, inst);
4338         if (r != 0 && r != ECONNABORTED)
4339                 bad_error("refresh_vertex", r);
4340         return (r);
4341 }
4342 
4343 /*
4344  * Returns true only if none of this service's dependents are 'up' -- online
4345  * or degraded (offline is considered down in this situation). This function
4346  * is somehow similar to is_nonsubgraph_leaf() but works on subtrees.
4347  */
4348 static boolean_t
4349 insubtree_dependents_down(graph_vertex_t *v)
4350 {
4351         graph_vertex_t *vv;
4352         graph_edge_t *e;
4353 
4354         assert(MUTEX_HELD(&dgraph_lock));
4355 
4356         for (e = uu_list_first(v->gv_dependents); e != NULL;
4357             e = uu_list_next(v->gv_dependents, e)) {
4358                 vv = e->ge_vertex;
4359                 if (vv->gv_type == GVT_INST) {
4360                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4361                                 continue;
4362 
4363                         if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4364                                 continue;
4365 
4366                         if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4367                             (vv->gv_state == RESTARTER_STATE_DEGRADED))
4368                                 return (B_FALSE);
4369                 } else {
4370                         /*
4371                          * Skip all excluded dependents and decide whether
4372                          * to offline the service based on the restart_on
4373                          * attribute.
4374                          */
4375                         if (is_depgrp_bypassed(vv))
4376                                 continue;
4377 
4378                         /*
4379                          * For dependency groups or service vertices, keep
4380                          * traversing to see if instances are running.
4381                          */
4382                         if (insubtree_dependents_down(vv) == B_FALSE)
4383                                 return (B_FALSE);
4384                 }
4385         }
4386 
4387         return (B_TRUE);
4388 }
4389 
4390 /*
4391  * Returns true only if none of this service's dependents are 'up' -- online,
4392  * degraded, or offline.
4393  */
4394 static int
4395 is_nonsubgraph_leaf(graph_vertex_t *v)
4396 {
4397         graph_vertex_t *vv;
4398         graph_edge_t *e;
4399 
4400         assert(MUTEX_HELD(&dgraph_lock));
4401 
4402         for (e = uu_list_first(v->gv_dependents);
4403             e != NULL;
4404             e = uu_list_next(v->gv_dependents, e)) {
4405 
4406                 vv = e->ge_vertex;
4407                 if (vv->gv_type == GVT_INST) {
4408                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4409                                 continue;
4410 
4411                         if (vv->gv_flags & GV_INSUBGRAPH)
4412                                 continue;
4413 
4414                         if (up_state(vv->gv_state))
4415                                 return (0);
4416                 } else {
4417                         /*
4418                          * For dependency group or service vertices, keep
4419                          * traversing to see if instances are running.
4420                          *
4421                          * We should skip exclude_all dependencies otherwise
4422                          * the vertex will never be considered as a leaf
4423                          * if the dependent is offline. The main reason for
4424                          * this is that disable_nonsubgraph_leaves() skips
4425                          * exclusion dependencies.
4426                          */
4427                         if (vv->gv_type == GVT_GROUP &&
4428                             vv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4429                                 continue;
4430 
4431                         if (!is_nonsubgraph_leaf(vv))
4432                                 return (0);
4433                 }
4434         }
4435 
4436         return (1);
4437 }
4438 
4439 /*
4440  * Disable v temporarily.  Attempt to do this by setting its enabled override
4441  * property in the repository.  If that fails, send a _DISABLE command.
4442  * Returns 0 on success and ECONNABORTED if the repository connection is
4443  * broken.
4444  */
4445 static int
4446 disable_service_temporarily(graph_vertex_t *v, scf_handle_t *h)
4447 {
4448         const char * const emsg = "Could not temporarily disable %s because "
4449             "%s.  Will stop service anyways.  Repository status for the "
4450             "service may be inaccurate.\n";
4451         const char * const emsg_cbroken =
4452             "the repository connection was broken";
4453 
4454         scf_instance_t *inst;
4455         int r;
4456 
4457         inst = scf_instance_create(h);
4458         if (inst == NULL) {
4459                 char buf[100];
4460 
4461                 (void) snprintf(buf, sizeof (buf),
4462                     "scf_instance_create() failed (%s)",
4463                     scf_strerror(scf_error()));
4464                 log_error(LOG_WARNING, emsg, v->gv_name, buf);
4465 
4466                 graph_enable_by_vertex(v, 0, 0);
4467                 return (0);
4468         }
4469 
4470         r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
4471             NULL, NULL, SCF_DECODE_FMRI_EXACT);
4472         if (r != 0) {
4473                 switch (scf_error()) {
4474                 case SCF_ERROR_CONNECTION_BROKEN:
4475                         log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4476                         graph_enable_by_vertex(v, 0, 0);
4477                         return (ECONNABORTED);
4478 
4479                 case SCF_ERROR_NOT_FOUND:
4480                         return (0);
4481 
4482                 case SCF_ERROR_HANDLE_MISMATCH:
4483                 case SCF_ERROR_INVALID_ARGUMENT:
4484                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4485                 case SCF_ERROR_NOT_BOUND:
4486                 default:
4487                         bad_error("scf_handle_decode_fmri",
4488                             scf_error());
4489                 }
4490         }
4491 
4492         r = libscf_set_enable_ovr(inst, 0);
4493         switch (r) {
4494         case 0:
4495                 scf_instance_destroy(inst);
4496                 return (0);
4497 
4498         case ECANCELED:
4499                 scf_instance_destroy(inst);
4500                 return (0);
4501 
4502         case ECONNABORTED:
4503                 log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4504                 graph_enable_by_vertex(v, 0, 0);
4505                 return (ECONNABORTED);
4506 
4507         case EPERM:
4508                 log_error(LOG_WARNING, emsg, v->gv_name,
4509                     "the repository denied permission");
4510                 graph_enable_by_vertex(v, 0, 0);
4511                 return (0);
4512 
4513         case EROFS:
4514                 log_error(LOG_WARNING, emsg, v->gv_name,
4515                     "the repository is read-only");
4516                 graph_enable_by_vertex(v, 0, 0);
4517                 return (0);
4518 
4519         default:
4520                 bad_error("libscf_set_enable_ovr", r);
4521                 /* NOTREACHED */
4522         }
4523 }
4524 
4525 /*
4526  * Of the transitive instance dependencies of v, offline those which are
4527  * in the subtree and which are leaves (i.e., have no dependents which are
4528  * "up").
4529  */
4530 void
4531 offline_subtree_leaves(graph_vertex_t *v, void *arg)
4532 {
4533         assert(MUTEX_HELD(&dgraph_lock));
4534 
4535         /* If v isn't an instance, recurse on its dependencies. */
4536         if (v->gv_type != GVT_INST) {
4537                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4538                 return;
4539         }
4540 
4541         /*
4542          * If v is not in the subtree, so should all of its dependencies,
4543          * so do nothing.
4544          */
4545         if ((v->gv_flags & GV_TOOFFLINE) == 0)
4546                 return;
4547 
4548         /* If v isn't a leaf because it's already down, recurse. */
4549         if (!up_state(v->gv_state)) {
4550                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4551                 return;
4552         }
4553 
4554         /* if v is a leaf, offline it or disable it if it's the last one */
4555         if (insubtree_dependents_down(v) == B_TRUE) {
4556                 if (v->gv_flags & GV_TODISABLE)
4557                         vertex_send_event(v,
4558                             RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
4559                 else
4560                         offline_vertex(v);
4561         }
4562 }
4563 
4564 void
4565 graph_offline_subtree_leaves(graph_vertex_t *v, void *h)
4566 {
4567         graph_walk_dependencies(v, offline_subtree_leaves, (void *)h);
4568 }
4569 
4570 
4571 /*
4572  * Of the transitive instance dependencies of v, disable those which are not
4573  * in the subgraph and which are leaves (i.e., have no dependents which are
4574  * "up").
4575  */
4576 static void
4577 disable_nonsubgraph_leaves(graph_vertex_t *v, void *arg)
4578 {
4579         assert(MUTEX_HELD(&dgraph_lock));
4580 
4581         /*
4582          * We must skip exclusion dependencies because they are allowed to
4583          * complete dependency cycles.  This is correct because A's exclusion
4584          * dependency on B doesn't bear on the order in which they should be
4585          * stopped.  Indeed, the exclusion dependency should guarantee that
4586          * they are never online at the same time.
4587          */
4588         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4589                 return;
4590 
4591         /* If v isn't an instance, recurse on its dependencies. */
4592         if (v->gv_type != GVT_INST)
4593                 goto recurse;
4594 
4595         if ((v->gv_flags & GV_CONFIGURED) == 0)
4596                 /*
4597                  * Unconfigured instances should have no dependencies, but in
4598                  * case they ever get them,
4599                  */
4600                 goto recurse;
4601 
4602         /*
4603          * If v is in the subgraph, so should all of its dependencies, so do
4604          * nothing.
4605          */
4606         if (v->gv_flags & GV_INSUBGRAPH)
4607                 return;
4608 
4609         /* If v isn't a leaf because it's already down, recurse. */
4610         if (!up_state(v->gv_state))
4611                 goto recurse;
4612 
4613         /* If v is disabled but not down yet, be patient. */
4614         if ((v->gv_flags & GV_ENABLED) == 0)
4615                 return;
4616 
4617         /* If v is a leaf, disable it. */
4618         if (is_nonsubgraph_leaf(v))
4619                 (void) disable_service_temporarily(v, (scf_handle_t *)arg);
4620 
4621         return;
4622 
4623 recurse:
4624         graph_walk_dependencies(v, disable_nonsubgraph_leaves, arg);
4625 }
4626 
4627 static int
4628 stn_restarter_state(restarter_instance_state_t rstate)
4629 {
4630         static const struct statemap {
4631                 restarter_instance_state_t restarter_state;
4632                 int scf_state;
4633         } map[] = {
4634                 { RESTARTER_STATE_UNINIT, SCF_STATE_UNINIT },
4635                 { RESTARTER_STATE_MAINT, SCF_STATE_MAINT },
4636                 { RESTARTER_STATE_OFFLINE, SCF_STATE_OFFLINE },
4637                 { RESTARTER_STATE_DISABLED, SCF_STATE_DISABLED },
4638                 { RESTARTER_STATE_ONLINE, SCF_STATE_ONLINE },
4639                 { RESTARTER_STATE_DEGRADED, SCF_STATE_DEGRADED }
4640         };
4641 
4642         int i;
4643 
4644         for (i = 0; i < sizeof (map) / sizeof (map[0]); i++) {
4645                 if (rstate == map[i].restarter_state)
4646                         return (map[i].scf_state);
4647         }
4648 
4649         return (-1);
4650 }
4651 
4652 /*
4653  * State transition counters
4654  * Not incremented atomically - indicative only
4655  */
4656 static uint64_t stev_ct_maint;
4657 static uint64_t stev_ct_hwerr;
4658 static uint64_t stev_ct_service;
4659 static uint64_t stev_ct_global;
4660 static uint64_t stev_ct_noprefs;
4661 static uint64_t stev_ct_from_uninit;
4662 static uint64_t stev_ct_bad_state;
4663 static uint64_t stev_ct_ovr_prefs;
4664 
4665 static void
4666 dgraph_state_transition_notify(graph_vertex_t *v,
4667     restarter_instance_state_t old_state, restarter_str_t reason)
4668 {
4669         restarter_instance_state_t new_state = v->gv_state;
4670         int stn_transition, maint;
4671         int from, to;
4672         nvlist_t *attr;
4673         fmev_pri_t pri = FMEV_LOPRI;
4674         int raise = 0;
4675 
4676         if ((from = stn_restarter_state(old_state)) == -1 ||
4677             (to = stn_restarter_state(new_state)) == -1) {
4678                 stev_ct_bad_state++;
4679                 return;
4680         }
4681 
4682         stn_transition = from << 16 | to;
4683 
4684         maint = (to == SCF_STATE_MAINT || from == SCF_STATE_MAINT);
4685 
4686         if (maint) {
4687                 /*
4688                  * All transitions to/from maintenance state must raise
4689                  * an event.
4690                  */
4691                 raise++;
4692                 pri = FMEV_HIPRI;
4693                 stev_ct_maint++;
4694         } else if (reason == restarter_str_ct_ev_hwerr) {
4695                 /*
4696                  * All transitions caused by hardware fault must raise
4697                  * an event
4698                  */
4699                 raise++;
4700                 pri = FMEV_HIPRI;
4701                 stev_ct_hwerr++;
4702         } else if (stn_transition & v->gv_stn_tset) {
4703                 /*
4704                  * Specifically enabled event.
4705                  */
4706                 raise++;
4707                 stev_ct_service++;
4708         } else if (from == SCF_STATE_UNINIT) {
4709                 /*
4710                  * Only raise these if specifically selected above.
4711                  */
4712                 stev_ct_from_uninit++;
4713         } else if (stn_transition & stn_global &&
4714             (IS_ENABLED(v) == 1 || to == SCF_STATE_DISABLED)) {
4715                 raise++;
4716                 stev_ct_global++;
4717         } else {
4718                 stev_ct_noprefs++;
4719         }
4720 
4721         if (info_events_all) {
4722                 stev_ct_ovr_prefs++;
4723                 raise++;
4724         }
4725         if (!raise)
4726                 return;
4727 
4728         if (nvlist_alloc(&attr, NV_UNIQUE_NAME, 0) != 0 ||
4729             nvlist_add_string(attr, "fmri", v->gv_name) != 0 ||
4730             nvlist_add_uint32(attr, "reason-version",
4731             restarter_str_version()) || nvlist_add_string(attr, "reason-short",
4732             restarter_get_str_short(reason)) != 0 ||
4733             nvlist_add_string(attr, "reason-long",
4734             restarter_get_str_long(reason)) != 0 ||
4735             nvlist_add_int32(attr, "transition", stn_transition) != 0) {
4736                 log_framework(LOG_WARNING,
4737                     "FMEV: %s could not create nvlist for transition "
4738                     "event: %s\n", v->gv_name, strerror(errno));
4739                 nvlist_free(attr);
4740                 return;
4741         }
4742 
4743         if (fmev_rspublish_nvl(FMEV_RULESET_SMF, "state-transition",
4744             instance_state_str[new_state], pri, attr) != FMEV_SUCCESS) {
4745                 log_framework(LOG_DEBUG,
4746                     "FMEV: %s failed to publish transition event: %s\n",
4747                     v->gv_name, fmev_strerror(fmev_errno));
4748                 nvlist_free(attr);
4749         }
4750 }
4751 
4752 /*
4753  * Find the vertex for inst_name.  If it doesn't exist, return ENOENT.
4754  * Otherwise set its state to state.  If the instance has entered a state
4755  * which requires automatic action, take it (Uninitialized: do
4756  * dgraph_refresh_instance() without the snapshot update.  Disabled: if the
4757  * instance should be enabled, send _ENABLE.  Offline: if the instance should
4758  * be disabled, send _DISABLE, and if its dependencies are satisfied, send
4759  * _START.  Online, Degraded: if the instance wasn't running, update its start
4760  * snapshot.  Maintenance: no action.)
4761  *
4762  * Also fails with ECONNABORTED, or EINVAL if state is invalid.
4763  */
4764 static int
4765 dgraph_set_instance_state(scf_handle_t *h, const char *inst_name,
4766     protocol_states_t *states)
4767 {
4768         graph_vertex_t *v;
4769         int err = 0;
4770         restarter_instance_state_t old_state;
4771         restarter_instance_state_t state = states->ps_state;
4772         restarter_error_t serr = states->ps_err;
4773 
4774         MUTEX_LOCK(&dgraph_lock);
4775 
4776         v = vertex_get_by_name(inst_name);
4777         if (v == NULL) {
4778                 MUTEX_UNLOCK(&dgraph_lock);
4779                 return (ENOENT);
4780         }
4781 
4782         assert(v->gv_type == GVT_INST);
4783 
4784         switch (state) {
4785         case RESTARTER_STATE_UNINIT:
4786         case RESTARTER_STATE_DISABLED:
4787         case RESTARTER_STATE_OFFLINE:
4788         case RESTARTER_STATE_ONLINE:
4789         case RESTARTER_STATE_DEGRADED:
4790         case RESTARTER_STATE_MAINT:
4791                 break;
4792 
4793         default:
4794                 MUTEX_UNLOCK(&dgraph_lock);
4795                 return (EINVAL);
4796         }
4797 
4798         log_framework(LOG_DEBUG, "Graph noting %s %s -> %s.\n", v->gv_name,
4799             instance_state_str[v->gv_state], instance_state_str[state]);
4800 
4801         old_state = v->gv_state;
4802         v->gv_state = state;
4803 
4804         v->gv_reason = states->ps_reason;
4805         err = gt_transition(h, v, serr, old_state);
4806         if (err == 0 && v->gv_state != old_state) {
4807                 dgraph_state_transition_notify(v, old_state, states->ps_reason);
4808         }
4809 
4810         MUTEX_UNLOCK(&dgraph_lock);
4811         return (err);
4812 }
4813 
4814 /*
4815  * Handle state changes during milestone shutdown.  See
4816  * dgraph_set_milestone().  If the repository connection is broken,
4817  * ECONNABORTED will be returned, though a _DISABLE command will be sent for
4818  * the vertex anyway.
4819  */
4820 int
4821 vertex_subgraph_dependencies_shutdown(scf_handle_t *h, graph_vertex_t *v,
4822     restarter_instance_state_t old_state)
4823 {
4824         int was_up, now_up;
4825         int ret = 0;
4826 
4827         assert(v->gv_type == GVT_INST);
4828 
4829         /* Don't care if we're not going to a milestone. */
4830         if (milestone == NULL)
4831                 return (0);
4832 
4833         /* Don't care if we already finished coming down. */
4834         if (non_subgraph_svcs == 0)
4835                 return (0);
4836 
4837         /* Don't care if the service is in the subgraph. */
4838         if (v->gv_flags & GV_INSUBGRAPH)
4839                 return (0);
4840 
4841         /*
4842          * Update non_subgraph_svcs.  It is the number of non-subgraph
4843          * services which are in online, degraded, or offline.
4844          */
4845 
4846         was_up = up_state(old_state);
4847         now_up = up_state(v->gv_state);
4848 
4849         if (!was_up && now_up) {
4850                 ++non_subgraph_svcs;
4851         } else if (was_up && !now_up) {
4852                 --non_subgraph_svcs;
4853 
4854                 if (non_subgraph_svcs == 0) {
4855                         if (halting != -1) {
4856                                 do_uadmin();
4857                         } else if (go_single_user_mode || go_to_level1) {
4858                                 (void) startd_thread_create(single_user_thread,
4859                                     NULL);
4860                         }
4861                         return (0);
4862                 }
4863         }
4864 
4865         /* If this service is a leaf, it should be disabled. */
4866         if ((v->gv_flags & GV_ENABLED) && is_nonsubgraph_leaf(v)) {
4867                 int r;
4868 
4869                 r = disable_service_temporarily(v, h);
4870                 switch (r) {
4871                 case 0:
4872                         break;
4873 
4874                 case ECONNABORTED:
4875                         ret = ECONNABORTED;
4876                         break;
4877 
4878                 default:
4879                         bad_error("disable_service_temporarily", r);
4880                 }
4881         }
4882 
4883         /*
4884          * If the service just came down, propagate the disable to the newly
4885          * exposed leaves.
4886          */
4887         if (was_up && !now_up)
4888                 graph_walk_dependencies(v, disable_nonsubgraph_leaves,
4889                     (void *)h);
4890 
4891         return (ret);
4892 }
4893 
4894 /*
4895  * Decide whether to start up an sulogin thread after a service is
4896  * finished changing state.  Only need to do the full can_come_up()
4897  * evaluation if an instance is changing state, we're not halfway through
4898  * loading the thread, and we aren't shutting down or going to the single
4899  * user milestone.
4900  */
4901 void
4902 graph_transition_sulogin(restarter_instance_state_t state,
4903     restarter_instance_state_t old_state)
4904 {
4905         assert(MUTEX_HELD(&dgraph_lock));
4906 
4907         if (state != old_state && st->st_load_complete &&
4908             !go_single_user_mode && !go_to_level1 &&
4909             halting == -1) {
4910                 if (!sulogin_thread_running && !can_come_up()) {
4911                         (void) startd_thread_create(sulogin_thread, NULL);
4912                         sulogin_thread_running = B_TRUE;
4913                 }
4914         }
4915 }
4916 
4917 /*
4918  * Propagate a start, stop event, or a satisfiability event.
4919  *
4920  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4921  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4922  * full dependent graph to check for newly satisfied nodes.  This is
4923  * necessary for cases when non-direct dependents may be effected but direct
4924  * dependents may not (e.g. for optional_all evaluations, see the
4925  * propagate_satbility() comments).
4926  *
4927  * PROPAGATE_SAT should be used whenever a non-running service moves into
4928  * a state which can satisfy optional dependencies, like disabled or
4929  * maintenance.
4930  */
4931 void
4932 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4933     restarter_error_t rerr)
4934 {
4935         if (type == PROPAGATE_STOP) {
4936                 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4937         } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4938                 graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
4939 
4940                 if (type == PROPAGATE_SAT)
4941                         propagate_satbility(v);
4942         } else {
4943 #ifndef NDEBUG
4944                 uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4945                     __LINE__, type);
4946 #endif
4947                 abort();
4948         }
4949 }
4950 
4951 /*
4952  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4953  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4954  * all property group dependencies, and the dependency on the restarter,
4955  * disposing of vertices as appropriate.  If other vertices depend on this
4956  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4957  * returns 0.
4958  */
4959 static int
4960 dgraph_remove_instance(const char *fmri, scf_handle_t *h)
4961 {
4962         graph_vertex_t *v;
4963         graph_edge_t *e;
4964         uu_list_t *old_deps;
4965         int err;
4966 
4967         log_framework(LOG_DEBUG, "Graph engine: Removing %s.\n", fmri);
4968 
4969         MUTEX_LOCK(&dgraph_lock);
4970 
4971         v = vertex_get_by_name(fmri);
4972         if (v == NULL) {
4973                 MUTEX_UNLOCK(&dgraph_lock);
4974                 return (0);
4975         }
4976 
4977         /* Send restarter delete event. */
4978         if (v->gv_flags & GV_CONFIGURED)
4979                 graph_unset_restarter(v);
4980 
4981         if (milestone > MILESTONE_NONE) {
4982                 /*
4983                  * Make a list of v's current dependencies so we can
4984                  * reevaluate their GV_INSUBGRAPH flags after the dependencies
4985                  * are removed.
4986                  */
4987                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
4988 
4989                 err = uu_list_walk(v->gv_dependencies,
4990                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
4991                 assert(err == 0);
4992         }
4993 
4994         delete_instance_dependencies(v, B_TRUE);
4995 
4996         /*
4997          * Deleting an instance can both satisfy and unsatisfy dependencies,
4998          * depending on their type.  First propagate the stop as a RERR_RESTART
4999          * event -- deletion isn't a fault, just a normal stop.  This gives
5000          * dependent services the chance to do a clean shutdown.  Then, mark
5001          * the service as unconfigured and propagate the start event for the
5002          * optional_all dependencies that might have become satisfied.
5003          */
5004         graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
5005 
5006         v->gv_flags &= ~GV_CONFIGURED;
5007         v->gv_flags &= ~GV_DEATHROW;
5008 
5009         graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
5010         propagate_satbility(v);
5011 
5012         /*
5013          * If there are no (non-service) dependents, the vertex can be
5014          * completely removed.
5015          */
5016         if (v != milestone && v->gv_refs == 0 &&
5017             uu_list_numnodes(v->gv_dependents) == 1)
5018                 remove_inst_vertex(v);
5019 
5020         if (milestone > MILESTONE_NONE) {
5021                 void *cookie = NULL;
5022 
5023                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5024                         v = e->ge_vertex;
5025 
5026                         if (vertex_unref(v) == VERTEX_INUSE)
5027                                 while (eval_subgraph(v, h) == ECONNABORTED)
5028                                         libscf_handle_rebind(h);
5029 
5030                         startd_free(e, sizeof (*e));
5031                 }
5032 
5033                 uu_list_destroy(old_deps);
5034         }
5035 
5036         MUTEX_UNLOCK(&dgraph_lock);
5037 
5038         return (0);
5039 }
5040 
5041 /*
5042  * Return the eventual (maybe current) milestone in the form of a
5043  * legacy runlevel.
5044  */
5045 static char
5046 target_milestone_as_runlevel()
5047 {
5048         assert(MUTEX_HELD(&dgraph_lock));
5049 
5050         if (milestone == NULL)
5051                 return ('3');
5052         else if (milestone == MILESTONE_NONE)
5053                 return ('0');
5054 
5055         if (strcmp(milestone->gv_name, multi_user_fmri) == 0)
5056                 return ('2');
5057         else if (strcmp(milestone->gv_name, single_user_fmri) == 0)
5058                 return ('S');
5059         else if (strcmp(milestone->gv_name, multi_user_svr_fmri) == 0)
5060                 return ('3');
5061 
5062 #ifndef NDEBUG
5063         (void) fprintf(stderr, "%s:%d: Unknown milestone name \"%s\".\n",
5064             __FILE__, __LINE__, milestone->gv_name);
5065 #endif
5066         abort();
5067         /* NOTREACHED */
5068 }
5069 
5070 static struct {
5071         char    rl;
5072         int     sig;
5073 } init_sigs[] = {
5074         { 'S', SIGBUS },
5075         { '0', SIGINT },
5076         { '1', SIGQUIT },
5077         { '2', SIGILL },
5078         { '3', SIGTRAP },
5079         { '4', SIGIOT },
5080         { '5', SIGEMT },
5081         { '6', SIGFPE },
5082         { 0, 0 }
5083 };
5084 
5085 static void
5086 signal_init(char rl)
5087 {
5088         pid_t init_pid;
5089         int i;
5090 
5091         assert(MUTEX_HELD(&dgraph_lock));
5092 
5093         if (zone_getattr(getzoneid(), ZONE_ATTR_INITPID, &init_pid,
5094             sizeof (init_pid)) != sizeof (init_pid)) {
5095                 log_error(LOG_NOTICE, "Could not get pid to signal init.\n");
5096                 return;
5097         }
5098 
5099         for (i = 0; init_sigs[i].rl != 0; ++i)
5100                 if (init_sigs[i].rl == rl)
5101                         break;
5102 
5103         if (init_sigs[i].rl != 0) {
5104                 if (kill(init_pid, init_sigs[i].sig) != 0) {
5105                         switch (errno) {
5106                         case EPERM:
5107                         case ESRCH:
5108                                 log_error(LOG_NOTICE, "Could not signal init: "
5109                                     "%s.\n", strerror(errno));
5110                                 break;
5111 
5112                         case EINVAL:
5113                         default:
5114                                 bad_error("kill", errno);
5115                         }
5116                 }
5117         }
5118 }
5119 
5120 /*
5121  * This is called when one of the major milestones changes state, or when
5122  * init is signalled and tells us it was told to change runlevel.  We wait
5123  * to reach the milestone because this allows /etc/inittab entries to retain
5124  * some boot ordering: historically, entries could place themselves before/after
5125  * the running of /sbin/rcX scripts but we can no longer make the
5126  * distinction because the /sbin/rcX scripts no longer exist as punctuation
5127  * marks in /etc/inittab.
5128  *
5129  * Also, we only trigger an update when we reach the eventual target
5130  * milestone: without this, an /etc/inittab entry marked only for
5131  * runlevel 2 would be executed for runlevel 3, which is not how
5132  * /etc/inittab entries work.
5133  *
5134  * If we're single user coming online, then we set utmpx to the target
5135  * runlevel so that legacy scripts can work as expected.
5136  */
5137 static void
5138 graph_runlevel_changed(char rl, int online)
5139 {
5140         char trl;
5141 
5142         assert(MUTEX_HELD(&dgraph_lock));
5143 
5144         trl = target_milestone_as_runlevel();
5145 
5146         if (online) {
5147                 if (rl == trl) {
5148                         current_runlevel = trl;
5149                         signal_init(trl);
5150                 } else if (rl == 'S') {
5151                         /*
5152                          * At boot, set the entry early for the benefit of the
5153                          * legacy init scripts.
5154                          */
5155                         utmpx_set_runlevel(trl, 'S', B_FALSE);
5156                 }
5157         } else {
5158                 if (rl == '3' && trl == '2') {
5159                         current_runlevel = trl;
5160                         signal_init(trl);
5161                 } else if (rl == '2' && trl == 'S') {
5162                         current_runlevel = trl;
5163                         signal_init(trl);
5164                 }
5165         }
5166 }
5167 
5168 /*
5169  * Move to a backwards-compatible runlevel by executing the appropriate
5170  * /etc/rc?.d/K* scripts and/or setting the milestone.
5171  *
5172  * Returns
5173  *   0 - success
5174  *   ECONNRESET - success, but handle was reset
5175  *   ECONNABORTED - repository connection broken
5176  *   ECANCELED - pg was deleted
5177  */
5178 static int
5179 dgraph_set_runlevel(scf_propertygroup_t *pg, scf_property_t *prop)
5180 {
5181         char rl;
5182         scf_handle_t *h;
5183         int r;
5184         const char *ms = NULL;  /* what to commit as options/milestone */
5185         boolean_t rebound = B_FALSE;
5186         int mark_rl = 0;
5187 
5188         const char * const stop = "stop";
5189 
5190         r = libscf_extract_runlevel(prop, &rl);
5191         switch (r) {
5192         case 0:
5193                 break;
5194 
5195         case ECONNABORTED:
5196         case ECANCELED:
5197                 return (r);
5198 
5199         case EINVAL:
5200         case ENOENT:
5201                 log_error(LOG_WARNING, "runlevel property is misconfigured; "
5202                     "ignoring.\n");
5203                 /* delete the bad property */
5204                 goto nolock_out;
5205 
5206         default:
5207                 bad_error("libscf_extract_runlevel", r);
5208         }
5209 
5210         switch (rl) {
5211         case 's':
5212                 rl = 'S';
5213                 /* FALLTHROUGH */
5214 
5215         case 'S':
5216         case '2':
5217         case '3':
5218                 /*
5219                  * These cases cause a milestone change, so
5220                  * graph_runlevel_changed() will eventually deal with
5221                  * signalling init.
5222                  */
5223                 break;
5224 
5225         case '0':
5226         case '1':
5227         case '4':
5228         case '5':
5229         case '6':
5230                 mark_rl = 1;
5231                 break;
5232 
5233         default:
5234                 log_framework(LOG_NOTICE, "Unknown runlevel '%c'.\n", rl);
5235                 ms = NULL;
5236                 goto nolock_out;
5237         }
5238 
5239         h = scf_pg_handle(pg);
5240 
5241         MUTEX_LOCK(&dgraph_lock);
5242 
5243         /*
5244          * Since this triggers no milestone changes, force it by hand.
5245          */
5246         if (current_runlevel == '4' && rl == '3')
5247                 mark_rl = 1;
5248 
5249         /*
5250          * 1. If we are here after an "init X":
5251          *
5252          * init X
5253          *      init/lscf_set_runlevel()
5254          *              process_pg_event()
5255          *              dgraph_set_runlevel()
5256          *
5257          * then we haven't passed through graph_runlevel_changed() yet,
5258          * therefore 'current_runlevel' has not changed for sure but 'rl' has.
5259          * In consequence, if 'rl' is lower than 'current_runlevel', we change
5260          * the system runlevel and execute the appropriate /etc/rc?.d/K* scripts
5261          * past this test.
5262          *
5263          * 2. On the other hand, if we are here after a "svcadm milestone":
5264          *
5265          * svcadm milestone X
5266          *      dgraph_set_milestone()
5267          *              handle_graph_update_event()
5268          *              dgraph_set_instance_state()
5269          *              graph_post_X_[online|offline]()
5270          *              graph_runlevel_changed()
5271          *              signal_init()
5272          *                      init/lscf_set_runlevel()
5273          *                              process_pg_event()
5274          *                              dgraph_set_runlevel()
5275          *
5276          * then we already passed through graph_runlevel_changed() (by the way
5277          * of dgraph_set_milestone()) and 'current_runlevel' may have changed
5278          * and already be equal to 'rl' so we are going to return immediately
5279          * from dgraph_set_runlevel() without changing the system runlevel and
5280          * without executing the /etc/rc?.d/K* scripts.
5281          */
5282         if (rl == current_runlevel) {
5283                 ms = NULL;
5284                 goto out;
5285         }
5286 
5287         log_framework(LOG_DEBUG, "Changing to runlevel '%c'.\n", rl);
5288 
5289         /*
5290          * Make sure stop rc scripts see the new settings via who -r.
5291          */
5292         utmpx_set_runlevel(rl, current_runlevel, B_TRUE);
5293 
5294         /*
5295          * Some run levels don't have a direct correspondence to any
5296          * milestones, so we have to signal init directly.
5297          */
5298         if (mark_rl) {
5299                 current_runlevel = rl;
5300                 signal_init(rl);
5301         }
5302 
5303         switch (rl) {
5304         case 'S':
5305                 uu_warn("The system is coming down for administration.  "
5306                     "Please wait.\n");
5307                 fork_rc_script(rl, stop, B_FALSE);
5308                 ms = single_user_fmri;
5309                 go_single_user_mode = B_TRUE;
5310                 break;
5311 
5312         case '0':
5313                 halting_time = time(NULL);
5314                 fork_rc_script(rl, stop, B_TRUE);
5315                 halting = AD_HALT;
5316                 goto uadmin;
5317 
5318         case '5':
5319                 halting_time = time(NULL);
5320                 fork_rc_script(rl, stop, B_TRUE);
5321                 halting = AD_POWEROFF;
5322                 goto uadmin;
5323 
5324         case '6':
5325                 halting_time = time(NULL);
5326                 fork_rc_script(rl, stop, B_TRUE);
5327                 if (scf_is_fastboot_default() && getzoneid() == GLOBAL_ZONEID)
5328                         halting = AD_FASTREBOOT;
5329                 else
5330                         halting = AD_BOOT;
5331 
5332 uadmin:
5333                 uu_warn("The system is coming down.  Please wait.\n");
5334                 ms = "none";
5335 
5336                 /*
5337                  * We can't wait until all services are offline since this
5338                  * thread is responsible for taking them offline.  Instead we
5339                  * set halting to the second argument for uadmin() and call
5340                  * do_uadmin() from dgraph_set_instance_state() when
5341                  * appropriate.
5342                  */
5343                 break;
5344 
5345         case '1':
5346                 if (current_runlevel != 'S') {
5347                         uu_warn("Changing to state 1.\n");
5348                         fork_rc_script(rl, stop, B_FALSE);
5349                 } else {
5350                         uu_warn("The system is coming up for administration.  "
5351                             "Please wait.\n");
5352                 }
5353                 ms = single_user_fmri;
5354                 go_to_level1 = B_TRUE;
5355                 break;
5356 
5357         case '2':
5358                 if (current_runlevel == '3' || current_runlevel == '4')
5359                         fork_rc_script(rl, stop, B_FALSE);
5360                 ms = multi_user_fmri;
5361                 break;
5362 
5363         case '3':
5364         case '4':
5365                 ms = "all";
5366                 break;
5367 
5368         default:
5369 #ifndef NDEBUG
5370                 (void) fprintf(stderr, "%s:%d: Uncaught case %d ('%c').\n",
5371                     __FILE__, __LINE__, rl, rl);
5372 #endif
5373                 abort();
5374         }
5375 
5376 out:
5377         MUTEX_UNLOCK(&dgraph_lock);
5378 
5379 nolock_out:
5380         switch (r = libscf_clear_runlevel(pg, ms)) {
5381         case 0:
5382                 break;
5383 
5384         case ECONNABORTED:
5385                 libscf_handle_rebind(h);
5386                 rebound = B_TRUE;
5387                 goto nolock_out;
5388 
5389         case ECANCELED:
5390                 break;
5391 
5392         case EPERM:
5393         case EACCES:
5394         case EROFS:
5395                 log_error(LOG_NOTICE, "Could not delete \"%s/%s\" property: "
5396                     "%s.\n", SCF_PG_OPTIONS, "runlevel", strerror(r));
5397                 break;
5398 
5399         default:
5400                 bad_error("libscf_clear_runlevel", r);
5401         }
5402 
5403         return (rebound ? ECONNRESET : 0);
5404 }
5405 
5406 /*
5407  * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5408  * to the instances that are supposed to go offline during an
5409  * administrative disable operation.
5410  */
5411 static int
5412 mark_subtree(graph_edge_t *e, void *arg)
5413 {
5414         graph_vertex_t *v;
5415         int r;
5416 
5417         v = e->ge_vertex;
5418 
5419         /* If it's already in the subgraph, skip. */
5420         if (v->gv_flags & GV_TOOFFLINE)
5421                 return (UU_WALK_NEXT);
5422 
5423         switch (v->gv_type) {
5424         case GVT_INST:
5425                 /* If the instance is already offline, skip it. */
5426                 if (!inst_running(v))
5427                         return (UU_WALK_NEXT);
5428 
5429                 v->gv_flags |= GV_TOOFFLINE;
5430                 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5431                 break;
5432         case GVT_GROUP:
5433                 /*
5434                  * Skip all excluded dependents and decide whether to offline
5435                  * the service based on the restart_on attribute.
5436                  */
5437                 if (is_depgrp_bypassed(v))
5438                         return (UU_WALK_NEXT);
5439                 break;
5440         }
5441 
5442         r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5443             0);
5444         assert(r == 0);
5445         return (UU_WALK_NEXT);
5446 }
5447 
5448 static int
5449 mark_subgraph(graph_edge_t *e, void *arg)
5450 {
5451         graph_vertex_t *v;
5452         int r;
5453         int optional = (int)arg;
5454 
5455         v = e->ge_vertex;
5456 
5457         /* If it's already in the subgraph, skip. */
5458         if (v->gv_flags & GV_INSUBGRAPH)
5459                 return (UU_WALK_NEXT);
5460 
5461         /*
5462          * Keep track if walk has entered an optional dependency group
5463          */
5464         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_OPTIONAL_ALL) {
5465                 optional = 1;
5466         }
5467         /*
5468          * Quit if we are in an optional dependency group and the instance
5469          * is disabled
5470          */
5471         if (optional && (v->gv_type == GVT_INST) &&
5472             (!(v->gv_flags & GV_ENBLD_NOOVR)))
5473                 return (UU_WALK_NEXT);
5474 
5475         v->gv_flags |= GV_INSUBGRAPH;
5476 
5477         /* Skip all excluded dependencies. */
5478         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
5479                 return (UU_WALK_NEXT);
5480 
5481         r = uu_list_walk(v->gv_dependencies, (uu_walk_fn_t *)mark_subgraph,
5482             (void *)optional, 0);
5483         assert(r == 0);
5484         return (UU_WALK_NEXT);
5485 }
5486 
5487 /*
5488  * Bring down all services which are not dependencies of fmri.  The
5489  * dependencies of fmri (direct & indirect) will constitute the "subgraph",
5490  * and will have the GV_INSUBGRAPH flag set.  The rest must be brought down,
5491  * which means the state is "disabled", "maintenance", or "uninitialized".  We
5492  * could consider "offline" to be down, and refrain from sending start
5493  * commands for such services, but that's not strictly necessary, so we'll
5494  * decline to intrude on the state machine.  It would probably confuse users
5495  * anyway.
5496  *
5497  * The services should be brought down in reverse-dependency order, so we
5498  * can't do it all at once here.  We initiate by override-disabling the leaves
5499  * of the dependency tree -- those services which are up but have no
5500  * dependents which are up.  When they come down,
5501  * vertex_subgraph_dependencies_shutdown() will override-disable the newly
5502  * exposed leaves.  Perseverance will ensure completion.
5503  *
5504  * Sometimes we need to take action when the transition is complete, like
5505  * start sulogin or halt the system.  To tell when we're done, we initialize
5506  * non_subgraph_svcs here to be the number of services which need to come
5507  * down.  As each does, we decrement the counter.  When it hits zero, we take
5508  * the appropriate action.  See vertex_subgraph_dependencies_shutdown().
5509  *
5510  * In case we're coming up, we also remove any enable-overrides for the
5511  * services which are dependencies of fmri.
5512  *
5513  * If norepository is true, the function will not change the repository.
5514  *
5515  * The decision to change the system run level in accordance with the milestone
5516  * is taken in dgraph_set_runlevel().
5517  *
5518  * Returns
5519  *   0 - success
5520  *   ECONNRESET - success, but handle was rebound
5521  *   EINVAL - fmri is invalid (error is logged)
5522  *   EALREADY - the milestone is already set to fmri
5523  *   ENOENT - a configured vertex does not exist for fmri (an error is logged)
5524  */
5525 static int
5526 dgraph_set_milestone(const char *fmri, scf_handle_t *h, boolean_t norepository)
5527 {
5528         const char *cfmri, *fs;
5529         graph_vertex_t *nm, *v;
5530         int ret = 0, r;
5531         scf_instance_t *inst;
5532         boolean_t isall, isnone, rebound = B_FALSE;
5533 
5534         /* Validate fmri */
5535         isall = (strcmp(fmri, "all") == 0);
5536         isnone = (strcmp(fmri, "none") == 0);
5537 
5538         if (!isall && !isnone) {
5539                 if (fmri_canonify(fmri, (char **)&cfmri, B_FALSE) == EINVAL)
5540                         goto reject;
5541 
5542                 if (strcmp(cfmri, single_user_fmri) != 0 &&
5543                     strcmp(cfmri, multi_user_fmri) != 0 &&
5544                     strcmp(cfmri, multi_user_svr_fmri) != 0) {
5545                         startd_free((void *)cfmri, max_scf_fmri_size);
5546 reject:
5547                         log_framework(LOG_WARNING,
5548                             "Rejecting request for invalid milestone \"%s\".\n",
5549                             fmri);
5550                         return (EINVAL);
5551                 }
5552         }
5553 
5554         inst = safe_scf_instance_create(h);
5555 
5556         MUTEX_LOCK(&dgraph_lock);
5557 
5558         if (milestone == NULL) {
5559                 if (isall) {
5560                         log_framework(LOG_DEBUG,
5561                             "Milestone already set to all.\n");
5562                         ret = EALREADY;
5563                         goto out;
5564                 }
5565         } else if (milestone == MILESTONE_NONE) {
5566                 if (isnone) {
5567                         log_framework(LOG_DEBUG,
5568                             "Milestone already set to none.\n");
5569                         ret = EALREADY;
5570                         goto out;
5571                 }
5572         } else {
5573                 if (!isall && !isnone &&
5574                     strcmp(cfmri, milestone->gv_name) == 0) {
5575                         log_framework(LOG_DEBUG,
5576                             "Milestone already set to %s.\n", cfmri);
5577                         ret = EALREADY;
5578                         goto out;
5579                 }
5580         }
5581 
5582         if (!isall && !isnone) {
5583                 nm = vertex_get_by_name(cfmri);
5584                 if (nm == NULL || !(nm->gv_flags & GV_CONFIGURED)) {
5585                         log_framework(LOG_WARNING, "Cannot set milestone to %s "
5586                             "because no such service exists.\n", cfmri);
5587                         ret = ENOENT;
5588                         goto out;
5589                 }
5590         }
5591 
5592         log_framework(LOG_DEBUG, "Changing milestone to %s.\n", fmri);
5593 
5594         /*
5595          * Set milestone, removing the old one if this was the last reference.
5596          */
5597         if (milestone > MILESTONE_NONE)
5598                 (void) vertex_unref(milestone);
5599 
5600         if (isall)
5601                 milestone = NULL;
5602         else if (isnone)
5603                 milestone = MILESTONE_NONE;
5604         else {
5605                 milestone = nm;
5606                 /* milestone should count as a reference */
5607                 vertex_ref(milestone);
5608         }
5609 
5610         /* Clear all GV_INSUBGRAPH bits. */
5611         for (v = uu_list_first(dgraph); v != NULL; v = uu_list_next(dgraph, v))
5612                 v->gv_flags &= ~GV_INSUBGRAPH;
5613 
5614         if (!isall && !isnone) {
5615                 /* Set GV_INSUBGRAPH for milestone & descendents. */
5616                 milestone->gv_flags |= GV_INSUBGRAPH;
5617 
5618                 r = uu_list_walk(milestone->gv_dependencies,
5619                     (uu_walk_fn_t *)mark_subgraph, NULL, 0);
5620                 assert(r == 0);
5621         }
5622 
5623         /* Un-override services in the subgraph & override-disable the rest. */
5624         if (norepository)
5625                 goto out;
5626 
5627         non_subgraph_svcs = 0;
5628         for (v = uu_list_first(dgraph);
5629             v != NULL;
5630             v = uu_list_next(dgraph, v)) {
5631                 if (v->gv_type != GVT_INST ||
5632                     (v->gv_flags & GV_CONFIGURED) == 0)
5633                         continue;
5634 
5635 again:
5636                 r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
5637                     NULL, NULL, SCF_DECODE_FMRI_EXACT);
5638                 if (r != 0) {
5639                         switch (scf_error()) {
5640                         case SCF_ERROR_CONNECTION_BROKEN:
5641                         default:
5642                                 libscf_handle_rebind(h);
5643                                 rebound = B_TRUE;
5644                                 goto again;
5645 
5646                         case SCF_ERROR_NOT_FOUND:
5647                                 continue;
5648 
5649                         case SCF_ERROR_HANDLE_MISMATCH:
5650                         case SCF_ERROR_INVALID_ARGUMENT:
5651                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5652                         case SCF_ERROR_NOT_BOUND:
5653                                 bad_error("scf_handle_decode_fmri",
5654                                     scf_error());
5655                         }
5656                 }
5657 
5658                 if (isall || (v->gv_flags & GV_INSUBGRAPH)) {
5659                         r = libscf_delete_enable_ovr(inst);
5660                         fs = "libscf_delete_enable_ovr";
5661                 } else {
5662                         assert(isnone || (v->gv_flags & GV_INSUBGRAPH) == 0);
5663 
5664                         /*
5665                          * Services which are up need to come down before
5666                          * we're done, but we can only disable the leaves
5667                          * here.
5668                          */
5669 
5670                         if (up_state(v->gv_state))
5671                                 ++non_subgraph_svcs;
5672 
5673                         /* If it's already disabled, don't bother. */
5674                         if ((v->gv_flags & GV_ENABLED) == 0)
5675                                 continue;
5676 
5677                         if (!is_nonsubgraph_leaf(v))
5678                                 continue;
5679 
5680                         r = libscf_set_enable_ovr(inst, 0);
5681                         fs = "libscf_set_enable_ovr";
5682                 }
5683                 switch (r) {
5684                 case 0:
5685                 case ECANCELED:
5686                         break;
5687 
5688                 case ECONNABORTED:
5689                         libscf_handle_rebind(h);
5690                         rebound = B_TRUE;
5691                         goto again;
5692 
5693                 case EPERM:
5694                 case EROFS:
5695                         log_error(LOG_WARNING,
5696                             "Could not set %s/%s for %s: %s.\n",
5697                             SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
5698                             v->gv_name, strerror(r));
5699                         break;
5700 
5701                 default:
5702                         bad_error(fs, r);
5703                 }
5704         }
5705 
5706         if (halting != -1) {
5707                 if (non_subgraph_svcs > 1)
5708                         uu_warn("%d system services are now being stopped.\n",
5709                             non_subgraph_svcs);
5710                 else if (non_subgraph_svcs == 1)
5711                         uu_warn("One system service is now being stopped.\n");
5712                 else if (non_subgraph_svcs == 0)
5713                         do_uadmin();
5714         }
5715 
5716         ret = rebound ? ECONNRESET : 0;
5717 
5718 out:
5719         MUTEX_UNLOCK(&dgraph_lock);
5720         if (!isall && !isnone)
5721                 startd_free((void *)cfmri, max_scf_fmri_size);
5722         scf_instance_destroy(inst);
5723         return (ret);
5724 }
5725 
5726 
5727 /*
5728  * Returns 0, ECONNABORTED, or EINVAL.
5729  */
5730 static int
5731 handle_graph_update_event(scf_handle_t *h, graph_protocol_event_t *e)
5732 {
5733         int r;
5734 
5735         switch (e->gpe_type) {
5736         case GRAPH_UPDATE_RELOAD_GRAPH:
5737                 log_error(LOG_WARNING,
5738                     "graph_event: reload graph unimplemented\n");
5739                 break;
5740 
5741         case GRAPH_UPDATE_STATE_CHANGE: {
5742                 protocol_states_t *states = e->gpe_data;
5743 
5744                 switch (r = dgraph_set_instance_state(h, e->gpe_inst, states)) {
5745                 case 0:
5746                 case ENOENT:
5747                         break;
5748 
5749                 case ECONNABORTED:
5750                         return (ECONNABORTED);
5751 
5752                 case EINVAL:
5753                 default:
5754 #ifndef NDEBUG
5755                         (void) fprintf(stderr, "dgraph_set_instance_state() "
5756                             "failed with unexpected error %d at %s:%d.\n", r,
5757                             __FILE__, __LINE__);
5758 #endif
5759                         abort();
5760                 }
5761 
5762                 startd_free(states, sizeof (protocol_states_t));
5763                 break;
5764         }
5765 
5766         default:
5767                 log_error(LOG_WARNING,
5768                     "graph_event_loop received an unknown event: %d\n",
5769                     e->gpe_type);
5770                 break;
5771         }
5772 
5773         return (0);
5774 }
5775 
5776 /*
5777  * graph_event_thread()
5778  *    Wait for state changes from the restarters.
5779  */
5780 /*ARGSUSED*/
5781 void *
5782 graph_event_thread(void *unused)
5783 {
5784         scf_handle_t *h;
5785         int err;
5786 
5787         (void) pthread_setname_np(pthread_self(), "graph_event");
5788 
5789         h = libscf_handle_create_bound_loop();
5790 
5791         /*CONSTCOND*/
5792         while (1) {
5793                 graph_protocol_event_t *e;
5794 
5795                 MUTEX_LOCK(&gu->gu_lock);
5796 
5797                 while (gu->gu_wakeup == 0)
5798                         (void) pthread_cond_wait(&gu->gu_cv, &gu->gu_lock);
5799 
5800                 gu->gu_wakeup = 0;
5801 
5802                 while ((e = graph_event_dequeue()) != NULL) {
5803                         MUTEX_LOCK(&e->gpe_lock);
5804                         MUTEX_UNLOCK(&gu->gu_lock);
5805 
5806                         while ((err = handle_graph_update_event(h, e)) ==
5807                             ECONNABORTED)
5808                                 libscf_handle_rebind(h);
5809 
5810                         if (err == 0)
5811                                 graph_event_release(e);
5812                         else
5813                                 graph_event_requeue(e);
5814 
5815                         MUTEX_LOCK(&gu->gu_lock);
5816                 }
5817 
5818                 MUTEX_UNLOCK(&gu->gu_lock);
5819         }
5820 
5821         /*
5822          * Unreachable for now -- there's currently no graceful cleanup
5823          * called on exit().
5824          */
5825         MUTEX_UNLOCK(&gu->gu_lock);
5826         scf_handle_destroy(h);
5827         return (NULL);
5828 }
5829 
5830 static void
5831 set_initial_milestone(scf_handle_t *h)
5832 {
5833         scf_instance_t *inst;
5834         char *fmri, *cfmri;
5835         size_t sz;
5836         int r;
5837 
5838         inst = safe_scf_instance_create(h);
5839         fmri = startd_alloc(max_scf_fmri_size);
5840 
5841         /*
5842          * If -m milestone= was specified, we want to set options_ovr/milestone
5843          * to it.  Otherwise we want to read what the milestone should be set
5844          * to.  Either way we need our inst.
5845          */
5846 get_self:
5847         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
5848             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5849                 switch (scf_error()) {
5850                 case SCF_ERROR_CONNECTION_BROKEN:
5851                         libscf_handle_rebind(h);
5852                         goto get_self;
5853 
5854                 case SCF_ERROR_NOT_FOUND:
5855                         if (st->st_subgraph != NULL &&
5856                             st->st_subgraph[0] != '\0') {
5857                                 sz = strlcpy(fmri, st->st_subgraph,
5858                                     max_scf_fmri_size);
5859                                 assert(sz < max_scf_fmri_size);
5860                         } else {
5861                                 fmri[0] = '\0';
5862                         }
5863                         break;
5864 
5865                 case SCF_ERROR_INVALID_ARGUMENT:
5866                 case SCF_ERROR_CONSTRAINT_VIOLATED:
5867                 case SCF_ERROR_HANDLE_MISMATCH:
5868                 default:
5869                         bad_error("scf_handle_decode_fmri", scf_error());
5870                 }
5871         } else {
5872                 if (st->st_subgraph != NULL && st->st_subgraph[0] != '\0') {
5873                         scf_propertygroup_t *pg;
5874 
5875                         pg = safe_scf_pg_create(h);
5876 
5877                         sz = strlcpy(fmri, st->st_subgraph, max_scf_fmri_size);
5878                         assert(sz < max_scf_fmri_size);
5879 
5880                         r = libscf_inst_get_or_add_pg(inst, SCF_PG_OPTIONS_OVR,
5881                             SCF_PG_OPTIONS_OVR_TYPE, SCF_PG_OPTIONS_OVR_FLAGS,
5882                             pg);
5883                         switch (r) {
5884                         case 0:
5885                                 break;
5886 
5887                         case ECONNABORTED:
5888                                 libscf_handle_rebind(h);
5889                                 goto get_self;
5890 
5891                         case EPERM:
5892                         case EACCES:
5893                         case EROFS:
5894                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5895                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5896                                     SCF_PROPERTY_MILESTONE, strerror(r));
5897                                 /* FALLTHROUGH */
5898 
5899                         case ECANCELED:
5900                                 sz = strlcpy(fmri, st->st_subgraph,
5901                                     max_scf_fmri_size);
5902                                 assert(sz < max_scf_fmri_size);
5903                                 break;
5904 
5905                         default:
5906                                 bad_error("libscf_inst_get_or_add_pg", r);
5907                         }
5908 
5909                         r = libscf_clear_runlevel(pg, fmri);
5910                         switch (r) {
5911                         case 0:
5912                                 break;
5913 
5914                         case ECONNABORTED:
5915                                 libscf_handle_rebind(h);
5916                                 goto get_self;
5917 
5918                         case EPERM:
5919                         case EACCES:
5920                         case EROFS:
5921                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5922                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5923                                     SCF_PROPERTY_MILESTONE, strerror(r));
5924                                 /* FALLTHROUGH */
5925 
5926                         case ECANCELED:
5927                                 sz = strlcpy(fmri, st->st_subgraph,
5928                                     max_scf_fmri_size);
5929                                 assert(sz < max_scf_fmri_size);
5930                                 break;
5931 
5932                         default:
5933                                 bad_error("libscf_clear_runlevel", r);
5934                         }
5935 
5936                         scf_pg_destroy(pg);
5937                 } else {
5938                         scf_property_t *prop;
5939                         scf_value_t *val;
5940 
5941                         prop = safe_scf_property_create(h);
5942                         val = safe_scf_value_create(h);
5943 
5944                         r = libscf_get_milestone(inst, prop, val, fmri,
5945                             max_scf_fmri_size);
5946                         switch (r) {
5947                         case 0:
5948                                 break;
5949 
5950                         case ECONNABORTED:
5951                                 libscf_handle_rebind(h);
5952                                 goto get_self;
5953 
5954                         case EINVAL:
5955                                 log_error(LOG_WARNING, "Milestone property is "
5956                                     "misconfigured.  Defaulting to \"all\".\n");
5957                                 /* FALLTHROUGH */
5958 
5959                         case ECANCELED:
5960                         case ENOENT:
5961                                 fmri[0] = '\0';
5962                                 break;
5963 
5964                         default:
5965                                 bad_error("libscf_get_milestone", r);
5966                         }
5967 
5968                         scf_value_destroy(val);
5969                         scf_property_destroy(prop);
5970                 }
5971         }
5972 
5973         if (fmri[0] == '\0' || strcmp(fmri, "all") == 0)
5974                 goto out;
5975 
5976         if (strcmp(fmri, "none") != 0) {
5977 retry:
5978                 if (scf_handle_decode_fmri(h, fmri, NULL, NULL, inst, NULL,
5979                     NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5980                         switch (scf_error()) {
5981                         case SCF_ERROR_INVALID_ARGUMENT:
5982                                 log_error(LOG_WARNING,
5983                                     "Requested milestone \"%s\" is invalid.  "
5984                                     "Reverting to \"all\".\n", fmri);
5985                                 goto out;
5986 
5987                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5988                                 log_error(LOG_WARNING, "Requested milestone "
5989                                     "\"%s\" does not specify an instance.  "
5990                                     "Reverting to \"all\".\n", fmri);
5991                                 goto out;
5992 
5993                         case SCF_ERROR_CONNECTION_BROKEN:
5994                                 libscf_handle_rebind(h);
5995                                 goto retry;
5996 
5997                         case SCF_ERROR_NOT_FOUND:
5998                                 log_error(LOG_WARNING, "Requested milestone "
5999                                     "\"%s\" not in repository.  Reverting to "
6000                                     "\"all\".\n", fmri);
6001                                 goto out;
6002 
6003                         case SCF_ERROR_HANDLE_MISMATCH:
6004                         default:
6005                                 bad_error("scf_handle_decode_fmri",
6006                                     scf_error());
6007                         }
6008                 }
6009 
6010                 r = fmri_canonify(fmri, &cfmri, B_FALSE);
6011                 assert(r == 0);
6012 
6013                 r = dgraph_add_instance(cfmri, inst, B_TRUE);
6014                 startd_free(cfmri, max_scf_fmri_size);
6015                 switch (r) {
6016                 case 0:
6017                         break;
6018 
6019                 case ECONNABORTED:
6020                         goto retry;
6021 
6022                 case EINVAL:
6023                         log_error(LOG_WARNING,
6024                             "Requested milestone \"%s\" is invalid.  "
6025                             "Reverting to \"all\".\n", fmri);
6026                         goto out;
6027 
6028                 case ECANCELED:
6029                         log_error(LOG_WARNING,
6030                             "Requested milestone \"%s\" not "
6031                             "in repository.  Reverting to \"all\".\n",
6032                             fmri);
6033                         goto out;
6034 
6035                 case EEXIST:
6036                 default:
6037                         bad_error("dgraph_add_instance", r);
6038                 }
6039         }
6040 
6041         log_console(LOG_INFO, "Booting to milestone \"%s\".\n", fmri);
6042 
6043         r = dgraph_set_milestone(fmri, h, B_FALSE);
6044         switch (r) {
6045         case 0:
6046         case ECONNRESET:
6047         case EALREADY:
6048                 break;
6049 
6050         case EINVAL:
6051         case ENOENT:
6052         default:
6053                 bad_error("dgraph_set_milestone", r);
6054         }
6055 
6056 out:
6057         startd_free(fmri, max_scf_fmri_size);
6058         scf_instance_destroy(inst);
6059 }
6060 
6061 void
6062 set_restart_milestone(scf_handle_t *h)
6063 {
6064         scf_instance_t *inst;
6065         scf_property_t *prop;
6066         scf_value_t *val;
6067         char *fmri;
6068         int r;
6069 
6070         inst = safe_scf_instance_create(h);
6071 
6072 get_self:
6073         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL,
6074             inst, NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
6075                 switch (scf_error()) {
6076                 case SCF_ERROR_CONNECTION_BROKEN:
6077                         libscf_handle_rebind(h);
6078                         goto get_self;
6079 
6080                 case SCF_ERROR_NOT_FOUND:
6081                         break;
6082 
6083                 case SCF_ERROR_INVALID_ARGUMENT:
6084                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6085                 case SCF_ERROR_HANDLE_MISMATCH:
6086                 default:
6087                         bad_error("scf_handle_decode_fmri", scf_error());
6088                 }
6089 
6090                 scf_instance_destroy(inst);
6091                 return;
6092         }
6093 
6094         prop = safe_scf_property_create(h);
6095         val = safe_scf_value_create(h);
6096         fmri = startd_alloc(max_scf_fmri_size);
6097 
6098         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6099         switch (r) {
6100         case 0:
6101                 break;
6102 
6103         case ECONNABORTED:
6104                 libscf_handle_rebind(h);
6105                 goto get_self;
6106 
6107         case ECANCELED:
6108         case ENOENT:
6109         case EINVAL:
6110                 goto out;
6111 
6112         default:
6113                 bad_error("libscf_get_milestone", r);
6114         }
6115 
6116         r = dgraph_set_milestone(fmri, h, B_TRUE);
6117         switch (r) {
6118         case 0:
6119         case ECONNRESET:
6120         case EALREADY:
6121         case EINVAL:
6122         case ENOENT:
6123                 break;
6124 
6125         default:
6126                 bad_error("dgraph_set_milestone", r);
6127         }
6128 
6129 out:
6130         startd_free(fmri, max_scf_fmri_size);
6131         scf_value_destroy(val);
6132         scf_property_destroy(prop);
6133         scf_instance_destroy(inst);
6134 }
6135 
6136 /*
6137  * void *graph_thread(void *)
6138  *
6139  * Graph management thread.
6140  */
6141 /*ARGSUSED*/
6142 void *
6143 graph_thread(void *arg)
6144 {
6145         scf_handle_t *h;
6146         int err;
6147 
6148         (void) pthread_setname_np(pthread_self(), "graph");
6149 
6150         h = libscf_handle_create_bound_loop();
6151 
6152         if (st->st_initial)
6153                 set_initial_milestone(h);
6154 
6155         MUTEX_LOCK(&dgraph_lock);
6156         initial_milestone_set = B_TRUE;
6157         err = pthread_cond_broadcast(&initial_milestone_cv);
6158         assert(err == 0);
6159         MUTEX_UNLOCK(&dgraph_lock);
6160 
6161         libscf_populate_graph(h);
6162 
6163         if (!st->st_initial)
6164                 set_restart_milestone(h);
6165 
6166         MUTEX_LOCK(&st->st_load_lock);
6167         st->st_load_complete = 1;
6168         (void) pthread_cond_broadcast(&st->st_load_cv);
6169         MUTEX_UNLOCK(&st->st_load_lock);
6170 
6171         MUTEX_LOCK(&dgraph_lock);
6172         /*
6173          * Now that we've set st_load_complete we need to check can_come_up()
6174          * since if we booted to a milestone, then there won't be any more
6175          * state updates.
6176          */
6177         if (!go_single_user_mode && !go_to_level1 &&
6178             halting == -1) {
6179                 if (!sulogin_thread_running && !can_come_up()) {
6180                         (void) startd_thread_create(sulogin_thread, NULL);
6181                         sulogin_thread_running = B_TRUE;
6182                 }
6183         }
6184         MUTEX_UNLOCK(&dgraph_lock);
6185 
6186         (void) pthread_mutex_lock(&gu->gu_freeze_lock);
6187 
6188         /*CONSTCOND*/
6189         while (1) {
6190                 (void) pthread_cond_wait(&gu->gu_freeze_cv,
6191                     &gu->gu_freeze_lock);
6192         }
6193 
6194         /*
6195          * Unreachable for now -- there's currently no graceful cleanup
6196          * called on exit().
6197          */
6198         (void) pthread_mutex_unlock(&gu->gu_freeze_lock);
6199         scf_handle_destroy(h);
6200 
6201         return (NULL);
6202 }
6203 
6204 
6205 /*
6206  * int next_action()
6207  *   Given an array of timestamps 'a' with 'num' elements, find the
6208  *   lowest non-zero timestamp and return its index. If there are no
6209  *   non-zero elements, return -1.
6210  */
6211 static int
6212 next_action(hrtime_t *a, int num)
6213 {
6214         hrtime_t t = 0;
6215         int i = 0, smallest = -1;
6216 
6217         for (i = 0; i < num; i++) {
6218                 if (t == 0) {
6219                         t = a[i];
6220                         smallest = i;
6221                 } else if (a[i] != 0 && a[i] < t) {
6222                         t = a[i];
6223                         smallest = i;
6224                 }
6225         }
6226 
6227         if (t == 0)
6228                 return (-1);
6229         else
6230                 return (smallest);
6231 }
6232 
6233 /*
6234  * void process_actions()
6235  *   Process actions requested by the administrator. Possibilities include:
6236  *   refresh, restart, maintenance mode off, maintenance mode on,
6237  *   maintenance mode immediate, and degraded.
6238  *
6239  *   The set of pending actions is represented in the repository as a
6240  *   per-instance property group, with each action being a single property
6241  *   in that group.  This property group is converted to an array, with each
6242  *   action type having an array slot.  The actions in the array at the
6243  *   time process_actions() is called are acted on in the order of the
6244  *   timestamp (which is the value stored in the slot).  A value of zero
6245  *   indicates that there is no pending action of the type associated with
6246  *   a particular slot.
6247  *
6248  *   Sending an action event multiple times before the restarter has a
6249  *   chance to process that action will force it to be run at the last
6250  *   timestamp where it appears in the ordering.
6251  *
6252  *   Turning maintenance mode on trumps all other actions.
6253  *
6254  *   Returns 0 or ECONNABORTED.
6255  */
6256 static int
6257 process_actions(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst)
6258 {
6259         scf_property_t *prop = NULL;
6260         scf_value_t *val = NULL;
6261         scf_type_t type;
6262         graph_vertex_t *vertex;
6263         admin_action_t a;
6264         int i, ret = 0, r;
6265         hrtime_t action_ts[NACTIONS];
6266         char *inst_name;
6267 
6268         r = libscf_instance_get_fmri(inst, &inst_name);
6269         switch (r) {
6270         case 0:
6271                 break;
6272 
6273         case ECONNABORTED:
6274                 return (ECONNABORTED);
6275 
6276         case ECANCELED:
6277                 return (0);
6278 
6279         default:
6280                 bad_error("libscf_instance_get_fmri", r);
6281         }
6282 
6283         MUTEX_LOCK(&dgraph_lock);
6284 
6285         vertex = vertex_get_by_name(inst_name);
6286         if (vertex == NULL) {
6287                 MUTEX_UNLOCK(&dgraph_lock);
6288                 log_framework(LOG_DEBUG, "%s: Can't find graph vertex. "
6289                     "The instance must have been removed.\n", inst_name);
6290                 startd_free(inst_name, max_scf_fmri_size);
6291                 return (0);
6292         }
6293 
6294         prop = safe_scf_property_create(h);
6295         val = safe_scf_value_create(h);
6296 
6297         for (i = 0; i < NACTIONS; i++) {
6298                 if (scf_pg_get_property(pg, admin_actions[i], prop) != 0) {
6299                         switch (scf_error()) {
6300                         case SCF_ERROR_CONNECTION_BROKEN:
6301                         default:
6302                                 ret = ECONNABORTED;
6303                                 goto out;
6304 
6305                         case SCF_ERROR_DELETED:
6306                                 goto out;
6307 
6308                         case SCF_ERROR_NOT_FOUND:
6309                                 action_ts[i] = 0;
6310                                 continue;
6311 
6312                         case SCF_ERROR_HANDLE_MISMATCH:
6313                         case SCF_ERROR_INVALID_ARGUMENT:
6314                         case SCF_ERROR_NOT_SET:
6315                                 bad_error("scf_pg_get_property", scf_error());
6316                         }
6317                 }
6318 
6319                 if (scf_property_type(prop, &type) != 0) {
6320                         switch (scf_error()) {
6321                         case SCF_ERROR_CONNECTION_BROKEN:
6322                         default:
6323                                 ret = ECONNABORTED;
6324                                 goto out;
6325 
6326                         case SCF_ERROR_DELETED:
6327                                 action_ts[i] = 0;
6328                                 continue;
6329 
6330                         case SCF_ERROR_NOT_SET:
6331                                 bad_error("scf_property_type", scf_error());
6332                         }
6333                 }
6334 
6335                 if (type != SCF_TYPE_INTEGER) {
6336                         action_ts[i] = 0;
6337                         continue;
6338                 }
6339 
6340                 if (scf_property_get_value(prop, val) != 0) {
6341                         switch (scf_error()) {
6342                         case SCF_ERROR_CONNECTION_BROKEN:
6343                         default:
6344                                 ret = ECONNABORTED;
6345                                 goto out;
6346 
6347                         case SCF_ERROR_DELETED:
6348                                 goto out;
6349 
6350                         case SCF_ERROR_NOT_FOUND:
6351                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6352                                 action_ts[i] = 0;
6353                                 continue;
6354 
6355                         case SCF_ERROR_NOT_SET:
6356                         case SCF_ERROR_PERMISSION_DENIED:
6357                                 bad_error("scf_property_get_value",
6358                                     scf_error());
6359                         }
6360                 }
6361 
6362                 r = scf_value_get_integer(val, &action_ts[i]);
6363                 assert(r == 0);
6364         }
6365 
6366         a = ADMIN_EVENT_MAINT_ON_IMMEDIATE;
6367         if (action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ||
6368             action_ts[ADMIN_EVENT_MAINT_ON]) {
6369                 a = action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ?
6370                     ADMIN_EVENT_MAINT_ON_IMMEDIATE : ADMIN_EVENT_MAINT_ON;
6371 
6372                 vertex_send_event(vertex, admin_events[a]);
6373                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6374                 switch (r) {
6375                 case 0:
6376                 case EACCES:
6377                         break;
6378 
6379                 case ECONNABORTED:
6380                         ret = ECONNABORTED;
6381                         goto out;
6382 
6383                 case EPERM:
6384                         uu_die("Insufficient privilege.\n");
6385                         /* NOTREACHED */
6386 
6387                 default:
6388                         bad_error("libscf_unset_action", r);
6389                 }
6390         }
6391 
6392         while ((a = next_action(action_ts, NACTIONS)) != -1) {
6393                 log_framework(LOG_DEBUG,
6394                     "Graph: processing %s action for %s.\n", admin_actions[a],
6395                     inst_name);
6396 
6397                 if (a == ADMIN_EVENT_REFRESH) {
6398                         r = dgraph_refresh_instance(vertex, inst);
6399                         switch (r) {
6400                         case 0:
6401                         case ECANCELED:
6402                         case EINVAL:
6403                         case -1:
6404                                 break;
6405 
6406                         case ECONNABORTED:
6407                                 /* pg & inst are reset now, so just return. */
6408                                 ret = ECONNABORTED;
6409                                 goto out;
6410 
6411                         default:
6412                                 bad_error("dgraph_refresh_instance", r);
6413                         }
6414                 }
6415 
6416                 vertex_send_event(vertex, admin_events[a]);
6417 
6418                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6419                 switch (r) {
6420                 case 0:
6421                 case EACCES:
6422                         break;
6423 
6424                 case ECONNABORTED:
6425                         ret = ECONNABORTED;
6426                         goto out;
6427 
6428                 case EPERM:
6429                         uu_die("Insufficient privilege.\n");
6430                         /* NOTREACHED */
6431 
6432                 default:
6433                         bad_error("libscf_unset_action", r);
6434                 }
6435 
6436                 action_ts[a] = 0;
6437         }
6438 
6439 out:
6440         MUTEX_UNLOCK(&dgraph_lock);
6441 
6442         scf_property_destroy(prop);
6443         scf_value_destroy(val);
6444         startd_free(inst_name, max_scf_fmri_size);
6445         return (ret);
6446 }
6447 
6448 /*
6449  * inst and pg_name are scratch space, and are unset on entry.
6450  * Returns
6451  *   0 - success
6452  *   ECONNRESET - success, but repository handle rebound
6453  *   ECONNABORTED - repository connection broken
6454  */
6455 static int
6456 process_pg_event(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst,
6457     char *pg_name)
6458 {
6459         int r;
6460         scf_property_t *prop;
6461         scf_value_t *val;
6462         char *fmri;
6463         boolean_t rebound = B_FALSE, rebind_inst = B_FALSE;
6464 
6465         if (scf_pg_get_name(pg, pg_name, max_scf_value_size) < 0) {
6466                 switch (scf_error()) {
6467                 case SCF_ERROR_CONNECTION_BROKEN:
6468                 default:
6469                         return (ECONNABORTED);
6470 
6471                 case SCF_ERROR_DELETED:
6472                         return (0);
6473 
6474                 case SCF_ERROR_NOT_SET:
6475                         bad_error("scf_pg_get_name", scf_error());
6476                 }
6477         }
6478 
6479         if (strcmp(pg_name, SCF_PG_GENERAL) == 0 ||
6480             strcmp(pg_name, SCF_PG_GENERAL_OVR) == 0) {
6481                 r = dgraph_update_general(pg);
6482                 switch (r) {
6483                 case 0:
6484                 case ENOTSUP:
6485                 case ECANCELED:
6486                         return (0);
6487 
6488                 case ECONNABORTED:
6489                         return (ECONNABORTED);
6490 
6491                 case -1:
6492                         /* Error should have been logged. */
6493                         return (0);
6494 
6495                 default:
6496                         bad_error("dgraph_update_general", r);
6497                 }
6498         } else if (strcmp(pg_name, SCF_PG_RESTARTER_ACTIONS) == 0) {
6499                 if (scf_pg_get_parent_instance(pg, inst) != 0) {
6500                         switch (scf_error()) {
6501                         case SCF_ERROR_CONNECTION_BROKEN:
6502                                 return (ECONNABORTED);
6503 
6504                         case SCF_ERROR_DELETED:
6505                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6506                                 /* Ignore commands on services. */
6507                                 return (0);
6508 
6509                         case SCF_ERROR_NOT_BOUND:
6510                         case SCF_ERROR_HANDLE_MISMATCH:
6511                         case SCF_ERROR_NOT_SET:
6512                         default:
6513                                 bad_error("scf_pg_get_parent_instance",
6514                                     scf_error());
6515                         }
6516                 }
6517 
6518                 return (process_actions(h, pg, inst));
6519         }
6520 
6521         if (strcmp(pg_name, SCF_PG_OPTIONS) != 0 &&
6522             strcmp(pg_name, SCF_PG_OPTIONS_OVR) != 0)
6523                 return (0);
6524 
6525         /*
6526          * We only care about the options[_ovr] property groups of our own
6527          * instance, so get the fmri and compare.  Plus, once we know it's
6528          * correct, if the repository connection is broken we know exactly what
6529          * property group we were operating on, and can look it up again.
6530          */
6531         if (scf_pg_get_parent_instance(pg, inst) != 0) {
6532                 switch (scf_error()) {
6533                 case SCF_ERROR_CONNECTION_BROKEN:
6534                         return (ECONNABORTED);
6535 
6536                 case SCF_ERROR_DELETED:
6537                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6538                         return (0);
6539 
6540                 case SCF_ERROR_HANDLE_MISMATCH:
6541                 case SCF_ERROR_NOT_BOUND:
6542                 case SCF_ERROR_NOT_SET:
6543                 default:
6544                         bad_error("scf_pg_get_parent_instance",
6545                             scf_error());
6546                 }
6547         }
6548 
6549         switch (r = libscf_instance_get_fmri(inst, &fmri)) {
6550         case 0:
6551                 break;
6552 
6553         case ECONNABORTED:
6554                 return (ECONNABORTED);
6555 
6556         case ECANCELED:
6557                 return (0);
6558 
6559         default:
6560                 bad_error("libscf_instance_get_fmri", r);
6561         }
6562 
6563         if (strcmp(fmri, SCF_SERVICE_STARTD) != 0) {
6564                 startd_free(fmri, max_scf_fmri_size);
6565                 return (0);
6566         }
6567 
6568         /*
6569          * update the information events flag
6570          */
6571         if (strcmp(pg_name, SCF_PG_OPTIONS) == 0)
6572                 info_events_all = libscf_get_info_events_all(pg);
6573 
6574         prop = safe_scf_property_create(h);
6575         val = safe_scf_value_create(h);
6576 
6577         if (strcmp(pg_name, SCF_PG_OPTIONS_OVR) == 0) {
6578                 /* See if we need to set the runlevel. */
6579                 /* CONSTCOND */
6580                 if (0) {
6581 rebind_pg:
6582                         libscf_handle_rebind(h);
6583                         rebound = B_TRUE;
6584 
6585                         r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6586                         switch (r) {
6587                         case 0:
6588                                 break;
6589 
6590                         case ECONNABORTED:
6591                                 goto rebind_pg;
6592 
6593                         case ENOENT:
6594                                 goto out;
6595 
6596                         case EINVAL:
6597                         case ENOTSUP:
6598                                 bad_error("libscf_lookup_instance", r);
6599                         }
6600 
6601                         if (scf_instance_get_pg(inst, pg_name, pg) != 0) {
6602                                 switch (scf_error()) {
6603                                 case SCF_ERROR_DELETED:
6604                                 case SCF_ERROR_NOT_FOUND:
6605                                         goto out;
6606 
6607                                 case SCF_ERROR_CONNECTION_BROKEN:
6608                                         goto rebind_pg;
6609 
6610                                 case SCF_ERROR_HANDLE_MISMATCH:
6611                                 case SCF_ERROR_NOT_BOUND:
6612                                 case SCF_ERROR_NOT_SET:
6613                                 case SCF_ERROR_INVALID_ARGUMENT:
6614                                 default:
6615                                         bad_error("scf_instance_get_pg",
6616                                             scf_error());
6617                                 }
6618                         }
6619                 }
6620 
6621                 if (scf_pg_get_property(pg, "runlevel", prop) == 0) {
6622                         r = dgraph_set_runlevel(pg, prop);
6623                         switch (r) {
6624                         case ECONNRESET:
6625                                 rebound = B_TRUE;
6626                                 rebind_inst = B_TRUE;
6627                                 /* FALLTHROUGH */
6628 
6629                         case 0:
6630                                 break;
6631 
6632                         case ECONNABORTED:
6633                                 goto rebind_pg;
6634 
6635                         case ECANCELED:
6636                                 goto out;
6637 
6638                         default:
6639                                 bad_error("dgraph_set_runlevel", r);
6640                         }
6641                 } else {
6642                         switch (scf_error()) {
6643                         case SCF_ERROR_CONNECTION_BROKEN:
6644                         default:
6645                                 goto rebind_pg;
6646 
6647                         case SCF_ERROR_DELETED:
6648                                 goto out;
6649 
6650                         case SCF_ERROR_NOT_FOUND:
6651                                 break;
6652 
6653                         case SCF_ERROR_INVALID_ARGUMENT:
6654                         case SCF_ERROR_HANDLE_MISMATCH:
6655                         case SCF_ERROR_NOT_BOUND:
6656                         case SCF_ERROR_NOT_SET:
6657                                 bad_error("scf_pg_get_property", scf_error());
6658                         }
6659                 }
6660         }
6661 
6662         if (rebind_inst) {
6663 lookup_inst:
6664                 r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6665                 switch (r) {
6666                 case 0:
6667                         break;
6668 
6669                 case ECONNABORTED:
6670                         libscf_handle_rebind(h);
6671                         rebound = B_TRUE;
6672                         goto lookup_inst;
6673 
6674                 case ENOENT:
6675                         goto out;
6676 
6677                 case EINVAL:
6678                 case ENOTSUP:
6679                         bad_error("libscf_lookup_instance", r);
6680                 }
6681         }
6682 
6683         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6684         switch (r) {
6685         case 0:
6686                 break;
6687 
6688         case ECONNABORTED:
6689                 libscf_handle_rebind(h);
6690                 rebound = B_TRUE;
6691                 goto lookup_inst;
6692 
6693         case EINVAL:
6694                 log_error(LOG_NOTICE,
6695                     "%s/%s property of %s is misconfigured.\n", pg_name,
6696                     SCF_PROPERTY_MILESTONE, SCF_SERVICE_STARTD);
6697                 /* FALLTHROUGH */
6698 
6699         case ECANCELED:
6700         case ENOENT:
6701                 (void) strcpy(fmri, "all");
6702                 break;
6703 
6704         default:
6705                 bad_error("libscf_get_milestone", r);
6706         }
6707 
6708         r = dgraph_set_milestone(fmri, h, B_FALSE);
6709         switch (r) {
6710         case 0:
6711         case ECONNRESET:
6712         case EALREADY:
6713                 break;
6714 
6715         case EINVAL:
6716                 log_error(LOG_WARNING, "Milestone %s is invalid.\n", fmri);
6717                 break;
6718 
6719         case ENOENT:
6720                 log_error(LOG_WARNING, "Milestone %s does not exist.\n", fmri);
6721                 break;
6722 
6723         default:
6724                 bad_error("dgraph_set_milestone", r);
6725         }
6726 
6727 out:
6728         startd_free(fmri, max_scf_fmri_size);
6729         scf_value_destroy(val);
6730         scf_property_destroy(prop);
6731 
6732         return (rebound ? ECONNRESET : 0);
6733 }
6734 
6735 /*
6736  * process_delete() deletes an instance from the dgraph if 'fmri' is an
6737  * instance fmri or if 'fmri' matches the 'general' property group of an
6738  * instance (or the 'general/enabled' property).
6739  *
6740  * 'fmri' may be overwritten and cannot be trusted on return by the caller.
6741  */
6742 static void
6743 process_delete(char *fmri, scf_handle_t *h)
6744 {
6745         char *lfmri, *end_inst_fmri;
6746         const char *inst_name = NULL;
6747         const char *pg_name = NULL;
6748         const char *prop_name = NULL;
6749 
6750         lfmri = safe_strdup(fmri);
6751 
6752         /* Determine if the FMRI is a property group or instance */
6753         if (scf_parse_svc_fmri(lfmri, NULL, NULL, &inst_name, &pg_name,
6754             &prop_name) != SCF_SUCCESS) {
6755                 log_error(LOG_WARNING,
6756                     "Received invalid FMRI \"%s\" from repository server.\n",
6757                     fmri);
6758         } else if (inst_name != NULL && pg_name == NULL) {
6759                 (void) dgraph_remove_instance(fmri, h);
6760         } else if (inst_name != NULL && pg_name != NULL) {
6761                 /*
6762                  * If we're deleting the 'general' property group or
6763                  * 'general/enabled' property then the whole instance
6764                  * must be removed from the dgraph.
6765                  */
6766                 if (strcmp(pg_name, SCF_PG_GENERAL) != 0) {
6767                         free(lfmri);
6768                         return;
6769                 }
6770 
6771                 if (prop_name != NULL &&
6772                     strcmp(prop_name, SCF_PROPERTY_ENABLED) != 0) {
6773                         free(lfmri);
6774                         return;
6775                 }
6776 
6777                 /*
6778                  * Because the instance has already been deleted from the
6779                  * repository, we cannot use any scf_ functions to retrieve
6780                  * the instance FMRI however we can easily reconstruct it
6781                  * manually.
6782                  */
6783                 end_inst_fmri = strstr(fmri, SCF_FMRI_PROPERTYGRP_PREFIX);
6784                 if (end_inst_fmri == NULL)
6785                         bad_error("process_delete", 0);
6786 
6787                 end_inst_fmri[0] = '\0';
6788 
6789                 (void) dgraph_remove_instance(fmri, h);
6790         }
6791 
6792         free(lfmri);
6793 }
6794 
6795 /*ARGSUSED*/
6796 void *
6797 repository_event_thread(void *unused)
6798 {
6799         scf_handle_t *h;
6800         scf_propertygroup_t *pg;
6801         scf_instance_t *inst;
6802         char *fmri = startd_alloc(max_scf_fmri_size);
6803         char *pg_name = startd_alloc(max_scf_value_size);
6804         int r;
6805 
6806         (void) pthread_setname_np(pthread_self(), "repository_event");
6807 
6808         h = libscf_handle_create_bound_loop();
6809 
6810         pg = safe_scf_pg_create(h);
6811         inst = safe_scf_instance_create(h);
6812 
6813 retry:
6814         if (_scf_notify_add_pgtype(h, SCF_GROUP_FRAMEWORK) != SCF_SUCCESS) {
6815                 if (scf_error() == SCF_ERROR_CONNECTION_BROKEN) {
6816                         libscf_handle_rebind(h);
6817                 } else {
6818                         log_error(LOG_WARNING,
6819                             "Couldn't set up repository notification "
6820                             "for property group type %s: %s\n",
6821                             SCF_GROUP_FRAMEWORK, scf_strerror(scf_error()));
6822 
6823                         (void) sleep(1);
6824                 }
6825 
6826                 goto retry;
6827         }
6828 
6829         /*CONSTCOND*/
6830         while (1) {
6831                 ssize_t res;
6832 
6833                 /* Note: fmri is only set on delete events. */
6834                 res = _scf_notify_wait(pg, fmri, max_scf_fmri_size);
6835                 if (res < 0) {
6836                         libscf_handle_rebind(h);
6837                         goto retry;
6838                 } else if (res == 0) {
6839                         /*
6840                          * property group modified.  inst and pg_name are
6841                          * pre-allocated scratch space.
6842                          */
6843                         if (scf_pg_update(pg) < 0) {
6844                                 switch (scf_error()) {
6845                                 case SCF_ERROR_DELETED:
6846                                         continue;
6847 
6848                                 case SCF_ERROR_CONNECTION_BROKEN:
6849                                         log_error(LOG_WARNING,
6850                                             "Lost repository event due to "
6851                                             "disconnection.\n");
6852                                         libscf_handle_rebind(h);
6853                                         goto retry;
6854 
6855                                 case SCF_ERROR_NOT_BOUND:
6856                                 case SCF_ERROR_NOT_SET:
6857                                 default:
6858                                         bad_error("scf_pg_update", scf_error());
6859                                 }
6860                         }
6861 
6862                         r = process_pg_event(h, pg, inst, pg_name);
6863                         switch (r) {
6864                         case 0:
6865                                 break;
6866 
6867                         case ECONNABORTED:
6868                                 log_error(LOG_WARNING, "Lost repository event "
6869                                     "due to disconnection.\n");
6870                                 libscf_handle_rebind(h);
6871                                 /* FALLTHROUGH */
6872 
6873                         case ECONNRESET:
6874                                 goto retry;
6875 
6876                         default:
6877                                 bad_error("process_pg_event", r);
6878                         }
6879                 } else {
6880                         /*
6881                          * Service, instance, or pg deleted.
6882                          * Don't trust fmri on return.
6883                          */
6884                         process_delete(fmri, h);
6885                 }
6886         }
6887 
6888         /*NOTREACHED*/
6889         return (NULL);
6890 }
6891 
6892 void
6893 graph_engine_start()
6894 {
6895         int err;
6896 
6897         (void) startd_thread_create(graph_thread, NULL);
6898 
6899         MUTEX_LOCK(&dgraph_lock);
6900         while (!initial_milestone_set) {
6901                 err = pthread_cond_wait(&initial_milestone_cv, &dgraph_lock);
6902                 assert(err == 0);
6903         }
6904         MUTEX_UNLOCK(&dgraph_lock);
6905 
6906         (void) startd_thread_create(repository_event_thread, NULL);
6907         (void) startd_thread_create(graph_event_thread, NULL);
6908 }