1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
  25  * Copyright 2016 RackTop Systems.
  26  */
  27 
  28 /*
  29  * graph.c - master restarter graph engine
  30  *
  31  *   The graph engine keeps a dependency graph of all service instances on the
  32  *   system, as recorded in the repository.  It decides when services should
  33  *   be brought up or down based on service states and dependencies and sends
  34  *   commands to restarters to effect any changes.  It also executes
  35  *   administrator commands sent by svcadm via the repository.
  36  *
  37  *   The graph is stored in uu_list_t *dgraph and its vertices are
  38  *   graph_vertex_t's, each of which has a name and an integer id unique to
  39  *   its name (see dict.c).  A vertex's type attribute designates the type
  40  *   of object it represents: GVT_INST for service instances, GVT_SVC for
  41  *   service objects (since service instances may depend on another service,
  42  *   rather than service instance), GVT_FILE for files (which services may
  43  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
  44  *   vertices are necessary because dependency lists may have particular
  45  *   grouping types (require any, require all, optional, or exclude) and
  46  *   event-propagation characteristics.
  47  *
  48  *   The initial graph is built by libscf_populate_graph() invoking
  49  *   dgraph_add_instance() for each instance in the repository.  The function
  50  *   adds a GVT_SVC vertex for the service if one does not already exist, adds
  51  *   a GVT_INST vertex named by the FMRI of the instance, and sets up the edges.
  52  *   The resulting web of vertices & edges associated with an instance's vertex
  53  *   includes
  54  *
  55  *     - an edge from the GVT_SVC vertex for the instance's service
  56  *
  57  *     - an edge to the GVT_INST vertex of the instance's resarter, if its
  58  *       restarter is not svc.startd
  59  *
  60  *     - edges from other GVT_INST vertices if the instance is a restarter
  61  *
  62  *     - for each dependency property group in the instance's "running"
  63  *       snapshot, an edge to a GVT_GROUP vertex named by the FMRI of the
  64  *       instance and the name of the property group
  65  *
  66  *     - for each value of the "entities" property in each dependency property
  67  *       group, an edge from the corresponding GVT_GROUP vertex to a
  68  *       GVT_INST, GVT_SVC, or GVT_FILE vertex
  69  *
  70  *     - edges from GVT_GROUP vertices for each dependent instance
  71  *
  72  *   After the edges are set up the vertex's GV_CONFIGURED flag is set.  If
  73  *   there are problems, or if a service is mentioned in a dependency but does
  74  *   not exist in the repository, the GV_CONFIGURED flag will be clear.
  75  *
  76  *   The graph and all of its vertices are protected by the dgraph_lock mutex.
  77  *   See restarter.c for more information.
  78  *
  79  *   The properties of an instance fall into two classes: immediate and
  80  *   snapshotted.  Immediate properties should have an immediate effect when
  81  *   changed.  Snapshotted properties should be read from a snapshot, so they
  82  *   only change when the snapshot changes.  The immediate properties used by
  83  *   the graph engine are general/enabled, general/restarter, and the properties
  84  *   in the restarter_actions property group.  Since they are immediate, they
  85  *   are not read out of a snapshot.  The snapshotted properties used by the
  86  *   graph engine are those in the property groups with type "dependency" and
  87  *   are read out of the "running" snapshot.  The "running" snapshot is created
  88  *   by the the graph engine as soon as possible, and it is updated, along with
  89  *   in-core copies of the data (dependency information for the graph engine) on
  90  *   receipt of the refresh command from svcadm.  In addition, the graph engine
  91  *   updates the "start" snapshot from the "running" snapshot whenever a service
  92  *   comes online.
  93  *
  94  *   When a DISABLE event is requested by the administrator, svc.startd shutdown
  95  *   the dependents first before shutting down the requested service.
  96  *   In graph_enable_by_vertex, we create a subtree that contains the dependent
  97  *   vertices by marking those vertices with the GV_TOOFFLINE flag. And we mark
  98  *   the vertex to disable with the GV_TODISABLE flag. Once the tree is created,
  99  *   we send the _ADMIN_DISABLE event to the leaves. The leaves will then
 100  *   transition from STATE_ONLINE/STATE_DEGRADED to STATE_OFFLINE/STATE_MAINT.
 101  *   In gt_enter_offline and gt_enter_maint if the vertex was in a subtree then
 102  *   we clear the GV_TOOFFLINE flag and walk the dependencies to offline the new
 103  *   exposed leaves. We do the same until we reach the last leaf (the one with
 104  *   the GV_TODISABLE flag). If the vertex to disable is also part of a larger
 105  *   subtree (eg. multiple DISABLE events on vertices in the same subtree) then
 106  *   once the first vertex is disabled (GV_TODISABLE flag is removed), we
 107  *   continue to propagate the offline event to the vertex's dependencies.
 108  *
 109  *
 110  * SMF state transition notifications
 111  *
 112  *   When an instance of a service managed by SMF changes state, svc.startd may
 113  *   publish a GPEC sysevent. All transitions to or from maintenance, a
 114  *   transition cause by a hardware error will generate an event.
 115  *   Other transitions will generate an event if there exist notification
 116  *   parameter for that transition. Notification parameters are stored in the
 117  *   SMF repository for the service/instance they refer to. System-wide
 118  *   notification parameters are stored in the global instance.
 119  *   svc.startd can be told to send events for all SMF state transitions despite
 120  *   of notification parameters by setting options/info_events_all to true in
 121  *   restarter:default
 122  *
 123  *   The set of transitions that generate events is cached in the
 124  *   dgraph_vertex_t gv_stn_tset for service/instance and in the global
 125  *   stn_global for the system-wide set. They are re-read when instances are
 126  *   refreshed.
 127  *
 128  *   The GPEC events published by svc.startd are consumed by fmd(1M). After
 129  *   processing these events, fmd(1M) publishes the processed events to
 130  *   notification agents. The notification agents read the notification
 131  *   parameters from the SMF repository through libscf(3LIB) interfaces and send
 132  *   the notification, or not, based on those parameters.
 133  *
 134  *   Subscription and publishing to the GPEC channels is done with the
 135  *   libfmevent(3LIB) wrappers fmev_[r]publish_*() and
 136  *   fmev_shdl_(un)subscribe().
 137  *
 138  */
 139 
 140 #include <sys/uadmin.h>
 141 #include <sys/wait.h>
 142 
 143 #include <assert.h>
 144 #include <errno.h>
 145 #include <fcntl.h>
 146 #include <fm/libfmevent.h>
 147 #include <libscf.h>
 148 #include <libscf_priv.h>
 149 #include <librestart.h>
 150 #include <libuutil.h>
 151 #include <locale.h>
 152 #include <poll.h>
 153 #include <pthread.h>
 154 #include <signal.h>
 155 #include <stddef.h>
 156 #include <stdio.h>
 157 #include <stdlib.h>
 158 #include <string.h>
 159 #include <strings.h>
 160 #include <sys/statvfs.h>
 161 #include <sys/uadmin.h>
 162 #include <zone.h>
 163 #if defined(__i386)
 164 #include <libgrubmgmt.h>
 165 #endif  /* __i386 */
 166 
 167 #include "startd.h"
 168 #include "protocol.h"
 169 
 170 
 171 #define MILESTONE_NONE  ((graph_vertex_t *)1)
 172 
 173 #define CONSOLE_LOGIN_FMRI      "svc:/system/console-login:default"
 174 #define FS_MINIMAL_FMRI         "svc:/system/filesystem/minimal:default"
 175 
 176 #define VERTEX_REMOVED  0       /* vertex has been freed  */
 177 #define VERTEX_INUSE    1       /* vertex is still in use */
 178 
 179 #define IS_ENABLED(v) ((v)->gv_flags & (GV_ENABLED | GV_ENBLD_NOOVR))
 180 
 181 /*
 182  * stn_global holds the tset for the system wide notification parameters.
 183  * It is updated on refresh of svc:/system/svc/global:default
 184  *
 185  * There are two assumptions that relax the need for a mutex:
 186  *     1. 32-bit value assignments are atomic
 187  *     2. Its value is consumed only in one point at
 188  *     dgraph_state_transition_notify(). There are no test and set races.
 189  *
 190  *     If either assumption is broken, we'll need a mutex to synchronize
 191  *     access to stn_global
 192  */
 193 int32_t stn_global;
 194 /*
 195  * info_events_all holds a flag to override notification parameters and send
 196  * Information events for all state transitions.
 197  * same about the need of a mutex here.
 198  */
 199 int info_events_all;
 200 
 201 /*
 202  * Services in these states are not considered 'down' by the
 203  * milestone/shutdown code.
 204  */
 205 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
 206         (state) == RESTARTER_STATE_DEGRADED || \
 207         (state) == RESTARTER_STATE_OFFLINE)
 208 
 209 #define is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
 210         ((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \
 211         (v->gv_restart < RERR_RESTART)))
 212 
 213 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
 214 static uu_list_t *dgraph;
 215 static pthread_mutex_t dgraph_lock;
 216 
 217 /*
 218  * milestone indicates the current subgraph.  When NULL, it is the entire
 219  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
 220  * services on which the target vertex depends.
 221  */
 222 static graph_vertex_t *milestone = NULL;
 223 static boolean_t initial_milestone_set = B_FALSE;
 224 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
 225 
 226 /* protected by dgraph_lock */
 227 static boolean_t sulogin_thread_running = B_FALSE;
 228 static boolean_t sulogin_running = B_FALSE;
 229 static boolean_t console_login_ready = B_FALSE;
 230 
 231 /* Number of services to come down to complete milestone transition. */
 232 static uint_t non_subgraph_svcs;
 233 
 234 /*
 235  * These variables indicate what should be done when we reach the milestone
 236  * target milestone, i.e., when non_subgraph_svcs == 0.  They are acted upon in
 237  * dgraph_set_instance_state().
 238  */
 239 static int halting = -1;
 240 static boolean_t go_single_user_mode = B_FALSE;
 241 static boolean_t go_to_level1 = B_FALSE;
 242 
 243 /*
 244  * Tracks when we started halting.
 245  */
 246 static time_t halting_time = 0;
 247 
 248 /*
 249  * This tracks the legacy runlevel to ensure we signal init and manage
 250  * utmpx entries correctly.
 251  */
 252 static char current_runlevel = '\0';
 253 
 254 /* Number of single user threads currently running */
 255 static pthread_mutex_t single_user_thread_lock;
 256 static int single_user_thread_count = 0;
 257 
 258 /* Statistics for dependency cycle-checking */
 259 static u_longlong_t dep_inserts = 0;
 260 static u_longlong_t dep_cycle_ns = 0;
 261 static u_longlong_t dep_insert_ns = 0;
 262 
 263 
 264 static const char * const emsg_invalid_restarter =
 265         "Transitioning %s to maintenance, restarter FMRI %s is invalid "
 266         "(see 'svcs -xv' for details).\n";
 267 static const char * const console_login_fmri = CONSOLE_LOGIN_FMRI;
 268 static const char * const single_user_fmri = SCF_MILESTONE_SINGLE_USER;
 269 static const char * const multi_user_fmri = SCF_MILESTONE_MULTI_USER;
 270 static const char * const multi_user_svr_fmri = SCF_MILESTONE_MULTI_USER_SERVER;
 271 
 272 
 273 /*
 274  * These services define the system being "up".  If none of them can come
 275  * online, then we will run sulogin on the console.  Note that the install ones
 276  * are for the miniroot and when installing CDs after the first.  can_come_up()
 277  * does the decision making, and an sulogin_thread() runs sulogin, which can be
 278  * started by dgraph_set_instance_state() or single_user_thread().
 279  *
 280  * NOTE: can_come_up() relies on SCF_MILESTONE_SINGLE_USER being the first
 281  * entry, which is only used when booting_to_single_user (boot -s) is set.
 282  * This is because when doing a "boot -s", sulogin is started from specials.c
 283  * after milestone/single-user comes online, for backwards compatibility.
 284  * In this case, SCF_MILESTONE_SINGLE_USER needs to be part of up_svcs
 285  * to ensure sulogin will be spawned if milestone/single-user cannot be reached.
 286  */
 287 static const char * const up_svcs[] = {
 288         SCF_MILESTONE_SINGLE_USER,
 289         CONSOLE_LOGIN_FMRI,
 290         "svc:/system/install-setup:default",
 291         "svc:/system/install:default",
 292         NULL
 293 };
 294 
 295 /* This array must have an element for each non-NULL element of up_svcs[]. */
 296 static graph_vertex_t *up_svcs_p[] = { NULL, NULL, NULL, NULL };
 297 
 298 /* These are for seed repository magic.  See can_come_up(). */
 299 static const char * const manifest_import = SCF_INSTANCE_MI;
 300 static graph_vertex_t *manifest_import_p = NULL;
 301 
 302 
 303 static char target_milestone_as_runlevel(void);
 304 static void graph_runlevel_changed(char rl, int online);
 305 static int dgraph_set_milestone(const char *, scf_handle_t *, boolean_t);
 306 static boolean_t should_be_in_subgraph(graph_vertex_t *v);
 307 static int mark_subtree(graph_edge_t *, void *);
 308 static boolean_t insubtree_dependents_down(graph_vertex_t *);
 309 
 310 /*
 311  * graph_vertex_compare()
 312  *      This function can compare either int *id or * graph_vertex_t *gv
 313  *      values, as the vertex id is always the first element of a
 314  *      graph_vertex structure.
 315  */
 316 /* ARGSUSED */
 317 static int
 318 graph_vertex_compare(const void *lc_arg, const void *rc_arg, void *private)
 319 {
 320         int lc_id = ((const graph_vertex_t *)lc_arg)->gv_id;
 321         int rc_id = *(int *)rc_arg;
 322 
 323         if (lc_id > rc_id)
 324                 return (1);
 325         if (lc_id < rc_id)
 326                 return (-1);
 327         return (0);
 328 }
 329 
 330 void
 331 graph_init()
 332 {
 333         graph_edge_pool = startd_list_pool_create("graph_edges",
 334             sizeof (graph_edge_t), offsetof(graph_edge_t, ge_link), NULL,
 335             UU_LIST_POOL_DEBUG);
 336         assert(graph_edge_pool != NULL);
 337 
 338         graph_vertex_pool = startd_list_pool_create("graph_vertices",
 339             sizeof (graph_vertex_t), offsetof(graph_vertex_t, gv_link),
 340             graph_vertex_compare, UU_LIST_POOL_DEBUG);
 341         assert(graph_vertex_pool != NULL);
 342 
 343         (void) pthread_mutex_init(&dgraph_lock, &mutex_attrs);
 344         (void) pthread_mutex_init(&single_user_thread_lock, &mutex_attrs);
 345         dgraph = startd_list_create(graph_vertex_pool, NULL, UU_LIST_SORTED);
 346         assert(dgraph != NULL);
 347 
 348         if (!st->st_initial)
 349                 current_runlevel = utmpx_get_runlevel();
 350 
 351         log_framework(LOG_DEBUG, "Initialized graph\n");
 352 }
 353 
 354 static graph_vertex_t *
 355 vertex_get_by_name(const char *name)
 356 {
 357         int id;
 358 
 359         assert(MUTEX_HELD(&dgraph_lock));
 360 
 361         id = dict_lookup_byname(name);
 362         if (id == -1)
 363                 return (NULL);
 364 
 365         return (uu_list_find(dgraph, &id, NULL, NULL));
 366 }
 367 
 368 static graph_vertex_t *
 369 vertex_get_by_id(int id)
 370 {
 371         assert(MUTEX_HELD(&dgraph_lock));
 372 
 373         if (id == -1)
 374                 return (NULL);
 375 
 376         return (uu_list_find(dgraph, &id, NULL, NULL));
 377 }
 378 
 379 /*
 380  * Creates a new vertex with the given name, adds it to the graph, and returns
 381  * a pointer to it.  The graph lock must be held by this thread on entry.
 382  */
 383 static graph_vertex_t *
 384 graph_add_vertex(const char *name)
 385 {
 386         int id;
 387         graph_vertex_t *v;
 388         void *p;
 389         uu_list_index_t idx;
 390 
 391         assert(MUTEX_HELD(&dgraph_lock));
 392 
 393         id = dict_insert(name);
 394 
 395         v = startd_zalloc(sizeof (*v));
 396 
 397         v->gv_id = id;
 398 
 399         v->gv_name = startd_alloc(strlen(name) + 1);
 400         (void) strcpy(v->gv_name, name);
 401 
 402         v->gv_dependencies = startd_list_create(graph_edge_pool, v, 0);
 403         v->gv_dependents = startd_list_create(graph_edge_pool, v, 0);
 404 
 405         p = uu_list_find(dgraph, &id, NULL, &idx);
 406         assert(p == NULL);
 407 
 408         uu_list_node_init(v, &v->gv_link, graph_vertex_pool);
 409         uu_list_insert(dgraph, v, idx);
 410 
 411         return (v);
 412 }
 413 
 414 /*
 415  * Removes v from the graph and frees it.  The graph should be locked by this
 416  * thread, and v should have no edges associated with it.
 417  */
 418 static void
 419 graph_remove_vertex(graph_vertex_t *v)
 420 {
 421         assert(MUTEX_HELD(&dgraph_lock));
 422 
 423         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 424         assert(uu_list_numnodes(v->gv_dependents) == 0);
 425         assert(v->gv_refs == 0);
 426 
 427         startd_free(v->gv_name, strlen(v->gv_name) + 1);
 428         uu_list_destroy(v->gv_dependencies);
 429         uu_list_destroy(v->gv_dependents);
 430         uu_list_remove(dgraph, v);
 431 
 432         startd_free(v, sizeof (graph_vertex_t));
 433 }
 434 
 435 static void
 436 graph_add_edge(graph_vertex_t *fv, graph_vertex_t *tv)
 437 {
 438         graph_edge_t *e, *re;
 439         int r;
 440 
 441         assert(MUTEX_HELD(&dgraph_lock));
 442 
 443         e = startd_alloc(sizeof (graph_edge_t));
 444         re = startd_alloc(sizeof (graph_edge_t));
 445 
 446         e->ge_parent = fv;
 447         e->ge_vertex = tv;
 448 
 449         re->ge_parent = tv;
 450         re->ge_vertex = fv;
 451 
 452         uu_list_node_init(e, &e->ge_link, graph_edge_pool);
 453         r = uu_list_insert_before(fv->gv_dependencies, NULL, e);
 454         assert(r == 0);
 455 
 456         uu_list_node_init(re, &re->ge_link, graph_edge_pool);
 457         r = uu_list_insert_before(tv->gv_dependents, NULL, re);
 458         assert(r == 0);
 459 }
 460 
 461 static void
 462 graph_remove_edge(graph_vertex_t *v, graph_vertex_t *dv)
 463 {
 464         graph_edge_t *e;
 465 
 466         for (e = uu_list_first(v->gv_dependencies);
 467             e != NULL;
 468             e = uu_list_next(v->gv_dependencies, e)) {
 469                 if (e->ge_vertex == dv) {
 470                         uu_list_remove(v->gv_dependencies, e);
 471                         startd_free(e, sizeof (graph_edge_t));
 472                         break;
 473                 }
 474         }
 475 
 476         for (e = uu_list_first(dv->gv_dependents);
 477             e != NULL;
 478             e = uu_list_next(dv->gv_dependents, e)) {
 479                 if (e->ge_vertex == v) {
 480                         uu_list_remove(dv->gv_dependents, e);
 481                         startd_free(e, sizeof (graph_edge_t));
 482                         break;
 483                 }
 484         }
 485 }
 486 
 487 static void
 488 remove_inst_vertex(graph_vertex_t *v)
 489 {
 490         graph_edge_t *e;
 491         graph_vertex_t *sv;
 492         int i;
 493 
 494         assert(MUTEX_HELD(&dgraph_lock));
 495         assert(uu_list_numnodes(v->gv_dependents) == 1);
 496         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 497         assert(v->gv_refs == 0);
 498         assert((v->gv_flags & GV_CONFIGURED) == 0);
 499 
 500         e = uu_list_first(v->gv_dependents);
 501         sv = e->ge_vertex;
 502         graph_remove_edge(sv, v);
 503 
 504         for (i = 0; up_svcs[i] != NULL; ++i) {
 505                 if (up_svcs_p[i] == v)
 506                         up_svcs_p[i] = NULL;
 507         }
 508 
 509         if (manifest_import_p == v)
 510                 manifest_import_p = NULL;
 511 
 512         graph_remove_vertex(v);
 513 
 514         if (uu_list_numnodes(sv->gv_dependencies) == 0 &&
 515             uu_list_numnodes(sv->gv_dependents) == 0 &&
 516             sv->gv_refs == 0)
 517                 graph_remove_vertex(sv);
 518 }
 519 
 520 static void
 521 graph_walk_dependents(graph_vertex_t *v, void (*func)(graph_vertex_t *, void *),
 522     void *arg)
 523 {
 524         graph_edge_t *e;
 525 
 526         for (e = uu_list_first(v->gv_dependents);
 527             e != NULL;
 528             e = uu_list_next(v->gv_dependents, e))
 529                 func(e->ge_vertex, arg);
 530 }
 531 
 532 static void
 533 graph_walk_dependencies(graph_vertex_t *v, void (*func)(graph_vertex_t *,
 534         void *), void *arg)
 535 {
 536         graph_edge_t *e;
 537 
 538         assert(MUTEX_HELD(&dgraph_lock));
 539 
 540         for (e = uu_list_first(v->gv_dependencies);
 541             e != NULL;
 542             e = uu_list_next(v->gv_dependencies, e)) {
 543 
 544                 func(e->ge_vertex, arg);
 545         }
 546 }
 547 
 548 /*
 549  * Generic graph walking function.
 550  *
 551  * Given a vertex, this function will walk either dependencies
 552  * (WALK_DEPENDENCIES) or dependents (WALK_DEPENDENTS) of a vertex recursively
 553  * for the entire graph.  It will avoid cycles and never visit the same vertex
 554  * twice.
 555  *
 556  * We avoid traversing exclusion dependencies, because they are allowed to
 557  * create cycles in the graph.  When propagating satisfiability, there is no
 558  * need to walk exclusion dependencies because exclude_all_satisfied() doesn't
 559  * test for satisfiability.
 560  *
 561  * The walker takes two callbacks.  The first is called before examining the
 562  * dependents of each vertex.  The second is called on each vertex after
 563  * examining its dependents.  This allows is_path_to() to construct a path only
 564  * after the target vertex has been found.
 565  */
 566 typedef enum {
 567         WALK_DEPENDENTS,
 568         WALK_DEPENDENCIES
 569 } graph_walk_dir_t;
 570 
 571 typedef int (*graph_walk_cb_t)(graph_vertex_t *, void *);
 572 
 573 typedef struct graph_walk_info {
 574         graph_walk_dir_t        gi_dir;
 575         uchar_t                 *gi_visited;    /* vertex bitmap */
 576         int                     (*gi_pre)(graph_vertex_t *, void *);
 577         void                    (*gi_post)(graph_vertex_t *, void *);
 578         void                    *gi_arg;        /* callback arg */
 579         int                     gi_ret;         /* return value */
 580 } graph_walk_info_t;
 581 
 582 static int
 583 graph_walk_recurse(graph_edge_t *e, graph_walk_info_t *gip)
 584 {
 585         uu_list_t *list;
 586         int r;
 587         graph_vertex_t *v = e->ge_vertex;
 588         int i;
 589         uint_t b;
 590 
 591         i = v->gv_id / 8;
 592         b = 1 << (v->gv_id % 8);
 593 
 594         /*
 595          * Check to see if we've visited this vertex already.
 596          */
 597         if (gip->gi_visited[i] & b)
 598                 return (UU_WALK_NEXT);
 599 
 600         gip->gi_visited[i] |= b;
 601 
 602         /*
 603          * Don't follow exclusions.
 604          */
 605         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
 606                 return (UU_WALK_NEXT);
 607 
 608         /*
 609          * Call pre-visit callback.  If this doesn't terminate the walk,
 610          * continue search.
 611          */
 612         if ((gip->gi_ret = gip->gi_pre(v, gip->gi_arg)) == UU_WALK_NEXT) {
 613                 /*
 614                  * Recurse using appropriate list.
 615                  */
 616                 if (gip->gi_dir == WALK_DEPENDENTS)
 617                         list = v->gv_dependents;
 618                 else
 619                         list = v->gv_dependencies;
 620 
 621                 r = uu_list_walk(list, (uu_walk_fn_t *)graph_walk_recurse,
 622                     gip, 0);
 623                 assert(r == 0);
 624         }
 625 
 626         /*
 627          * Callbacks must return either UU_WALK_NEXT or UU_WALK_DONE.
 628          */
 629         assert(gip->gi_ret == UU_WALK_NEXT || gip->gi_ret == UU_WALK_DONE);
 630 
 631         /*
 632          * If given a post-callback, call the function for every vertex.
 633          */
 634         if (gip->gi_post != NULL)
 635                 (void) gip->gi_post(v, gip->gi_arg);
 636 
 637         /*
 638          * Preserve the callback's return value.  If the callback returns
 639          * UU_WALK_DONE, then we propagate that to the caller in order to
 640          * terminate the walk.
 641          */
 642         return (gip->gi_ret);
 643 }
 644 
 645 static void
 646 graph_walk(graph_vertex_t *v, graph_walk_dir_t dir,
 647     int (*pre)(graph_vertex_t *, void *),
 648     void (*post)(graph_vertex_t *, void *), void *arg)
 649 {
 650         graph_walk_info_t gi;
 651         graph_edge_t fake;
 652         size_t sz = dictionary->dict_new_id / 8 + 1;
 653 
 654         gi.gi_visited = startd_zalloc(sz);
 655         gi.gi_pre = pre;
 656         gi.gi_post = post;
 657         gi.gi_arg = arg;
 658         gi.gi_dir = dir;
 659         gi.gi_ret = 0;
 660 
 661         /*
 662          * Fake up an edge for the first iteration
 663          */
 664         fake.ge_vertex = v;
 665         (void) graph_walk_recurse(&fake, &gi);
 666 
 667         startd_free(gi.gi_visited, sz);
 668 }
 669 
 670 typedef struct child_search {
 671         int     id;             /* id of vertex to look for */
 672         uint_t  depth;          /* recursion depth */
 673         /*
 674          * While the vertex is not found, path is NULL.  After the search, if
 675          * the vertex was found then path should point to a -1-terminated
 676          * array of vertex id's which constitute the path to the vertex.
 677          */
 678         int     *path;
 679 } child_search_t;
 680 
 681 static int
 682 child_pre(graph_vertex_t *v, void *arg)
 683 {
 684         child_search_t *cs = arg;
 685 
 686         cs->depth++;
 687 
 688         if (v->gv_id == cs->id) {
 689                 cs->path = startd_alloc((cs->depth + 1) * sizeof (int));
 690                 cs->path[cs->depth] = -1;
 691                 return (UU_WALK_DONE);
 692         }
 693 
 694         return (UU_WALK_NEXT);
 695 }
 696 
 697 static void
 698 child_post(graph_vertex_t *v, void *arg)
 699 {
 700         child_search_t *cs = arg;
 701 
 702         cs->depth--;
 703 
 704         if (cs->path != NULL)
 705                 cs->path[cs->depth] = v->gv_id;
 706 }
 707 
 708 /*
 709  * Look for a path from from to to.  If one exists, returns a pointer to
 710  * a NULL-terminated array of pointers to the vertices along the path.  If
 711  * there is no path, returns NULL.
 712  */
 713 static int *
 714 is_path_to(graph_vertex_t *from, graph_vertex_t *to)
 715 {
 716         child_search_t cs;
 717 
 718         cs.id = to->gv_id;
 719         cs.depth = 0;
 720         cs.path = NULL;
 721 
 722         graph_walk(from, WALK_DEPENDENCIES, child_pre, child_post, &cs);
 723 
 724         return (cs.path);
 725 }
 726 
 727 /*
 728  * Given an array of int's as returned by is_path_to, allocates a string of
 729  * their names joined by newlines.  Returns the size of the allocated buffer
 730  * in *sz and frees path.
 731  */
 732 static void
 733 path_to_str(int *path, char **cpp, size_t *sz)
 734 {
 735         int i;
 736         graph_vertex_t *v;
 737         size_t allocd, new_allocd;
 738         char *new, *name;
 739 
 740         assert(MUTEX_HELD(&dgraph_lock));
 741         assert(path[0] != -1);
 742 
 743         allocd = 1;
 744         *cpp = startd_alloc(1);
 745         (*cpp)[0] = '\0';
 746 
 747         for (i = 0; path[i] != -1; ++i) {
 748                 name = NULL;
 749 
 750                 v = vertex_get_by_id(path[i]);
 751 
 752                 if (v == NULL)
 753                         name = "<deleted>";
 754                 else if (v->gv_type == GVT_INST || v->gv_type == GVT_SVC)
 755                         name = v->gv_name;
 756 
 757                 if (name != NULL) {
 758                         new_allocd = allocd + strlen(name) + 1;
 759                         new = startd_alloc(new_allocd);
 760                         (void) strcpy(new, *cpp);
 761                         (void) strcat(new, name);
 762                         (void) strcat(new, "\n");
 763 
 764                         startd_free(*cpp, allocd);
 765 
 766                         *cpp = new;
 767                         allocd = new_allocd;
 768                 }
 769         }
 770 
 771         startd_free(path, sizeof (int) * (i + 1));
 772 
 773         *sz = allocd;
 774 }
 775 
 776 
 777 /*
 778  * This function along with run_sulogin() implements an exclusion relationship
 779  * between system/console-login and sulogin.  run_sulogin() will fail if
 780  * system/console-login is online, and the graph engine should call
 781  * graph_clogin_start() to bring system/console-login online, which defers the
 782  * start if sulogin is running.
 783  */
 784 static void
 785 graph_clogin_start(graph_vertex_t *v)
 786 {
 787         assert(MUTEX_HELD(&dgraph_lock));
 788 
 789         if (sulogin_running)
 790                 console_login_ready = B_TRUE;
 791         else
 792                 vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 793 }
 794 
 795 static void
 796 graph_su_start(graph_vertex_t *v)
 797 {
 798         /*
 799          * /etc/inittab used to have the initial /sbin/rcS as a 'sysinit'
 800          * entry with a runlevel of 'S', before jumping to the final
 801          * target runlevel (as set in initdefault).  We mimic that legacy
 802          * behavior here.
 803          */
 804         utmpx_set_runlevel('S', '0', B_FALSE);
 805         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 806 }
 807 
 808 static void
 809 graph_post_su_online(void)
 810 {
 811         graph_runlevel_changed('S', 1);
 812 }
 813 
 814 static void
 815 graph_post_su_disable(void)
 816 {
 817         graph_runlevel_changed('S', 0);
 818 }
 819 
 820 static void
 821 graph_post_mu_online(void)
 822 {
 823         graph_runlevel_changed('2', 1);
 824 }
 825 
 826 static void
 827 graph_post_mu_disable(void)
 828 {
 829         graph_runlevel_changed('2', 0);
 830 }
 831 
 832 static void
 833 graph_post_mus_online(void)
 834 {
 835         graph_runlevel_changed('3', 1);
 836 }
 837 
 838 static void
 839 graph_post_mus_disable(void)
 840 {
 841         graph_runlevel_changed('3', 0);
 842 }
 843 
 844 static struct special_vertex_info {
 845         const char      *name;
 846         void            (*start_f)(graph_vertex_t *);
 847         void            (*post_online_f)(void);
 848         void            (*post_disable_f)(void);
 849 } special_vertices[] = {
 850         { CONSOLE_LOGIN_FMRI, graph_clogin_start, NULL, NULL },
 851         { SCF_MILESTONE_SINGLE_USER, graph_su_start,
 852             graph_post_su_online, graph_post_su_disable },
 853         { SCF_MILESTONE_MULTI_USER, NULL,
 854             graph_post_mu_online, graph_post_mu_disable },
 855         { SCF_MILESTONE_MULTI_USER_SERVER, NULL,
 856             graph_post_mus_online, graph_post_mus_disable },
 857         { NULL },
 858 };
 859 
 860 
 861 void
 862 vertex_send_event(graph_vertex_t *v, restarter_event_type_t e)
 863 {
 864         switch (e) {
 865         case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
 866                 assert(v->gv_state == RESTARTER_STATE_UNINIT);
 867 
 868                 MUTEX_LOCK(&st->st_load_lock);
 869                 st->st_load_instances++;
 870                 MUTEX_UNLOCK(&st->st_load_lock);
 871                 break;
 872 
 873         case RESTARTER_EVENT_TYPE_ENABLE:
 874                 log_framework(LOG_DEBUG, "Enabling %s.\n", v->gv_name);
 875                 assert(v->gv_state == RESTARTER_STATE_UNINIT ||
 876                     v->gv_state == RESTARTER_STATE_DISABLED ||
 877                     v->gv_state == RESTARTER_STATE_MAINT);
 878                 break;
 879 
 880         case RESTARTER_EVENT_TYPE_DISABLE:
 881         case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
 882                 log_framework(LOG_DEBUG, "Disabling %s.\n", v->gv_name);
 883                 assert(v->gv_state != RESTARTER_STATE_DISABLED);
 884                 break;
 885 
 886         case RESTARTER_EVENT_TYPE_STOP_RESET:
 887         case RESTARTER_EVENT_TYPE_STOP:
 888                 log_framework(LOG_DEBUG, "Stopping %s.\n", v->gv_name);
 889                 assert(v->gv_state == RESTARTER_STATE_DEGRADED ||
 890                     v->gv_state == RESTARTER_STATE_ONLINE);
 891                 break;
 892 
 893         case RESTARTER_EVENT_TYPE_START:
 894                 log_framework(LOG_DEBUG, "Starting %s.\n", v->gv_name);
 895                 assert(v->gv_state == RESTARTER_STATE_OFFLINE);
 896                 break;
 897 
 898         case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
 899         case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
 900         case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
 901         case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
 902         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
 903         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
 904         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
 905         case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
 906         case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
 907                 break;
 908 
 909         default:
 910 #ifndef NDEBUG
 911                 uu_warn("%s:%d: Bad event %d.\n", __FILE__, __LINE__, e);
 912 #endif
 913                 abort();
 914         }
 915 
 916         restarter_protocol_send_event(v->gv_name, v->gv_restarter_channel, e,
 917             v->gv_reason);
 918 }
 919 
 920 static void
 921 graph_unset_restarter(graph_vertex_t *v)
 922 {
 923         assert(MUTEX_HELD(&dgraph_lock));
 924         assert(v->gv_flags & GV_CONFIGURED);
 925 
 926         vertex_send_event(v, RESTARTER_EVENT_TYPE_REMOVE_INSTANCE);
 927 
 928         if (v->gv_restarter_id != -1) {
 929                 graph_vertex_t *rv;
 930 
 931                 rv = vertex_get_by_id(v->gv_restarter_id);
 932                 graph_remove_edge(v, rv);
 933         }
 934 
 935         v->gv_restarter_id = -1;
 936         v->gv_restarter_channel = NULL;
 937 }
 938 
 939 /*
 940  * Return VERTEX_REMOVED when the vertex passed in argument is deleted from the
 941  * dgraph otherwise return VERTEX_INUSE.
 942  */
 943 static int
 944 free_if_unrefed(graph_vertex_t *v)
 945 {
 946         assert(MUTEX_HELD(&dgraph_lock));
 947 
 948         if (v->gv_refs > 0)
 949                 return (VERTEX_INUSE);
 950 
 951         if (v->gv_type == GVT_SVC &&
 952             uu_list_numnodes(v->gv_dependents) == 0 &&
 953             uu_list_numnodes(v->gv_dependencies) == 0) {
 954                 graph_remove_vertex(v);
 955                 return (VERTEX_REMOVED);
 956         } else if (v->gv_type == GVT_INST &&
 957             (v->gv_flags & GV_CONFIGURED) == 0 &&
 958             uu_list_numnodes(v->gv_dependents) == 1 &&
 959             uu_list_numnodes(v->gv_dependencies) == 0) {
 960                 remove_inst_vertex(v);
 961                 return (VERTEX_REMOVED);
 962         }
 963 
 964         return (VERTEX_INUSE);
 965 }
 966 
 967 static void
 968 delete_depgroup(graph_vertex_t *v)
 969 {
 970         graph_edge_t *e;
 971         graph_vertex_t *dv;
 972 
 973         assert(MUTEX_HELD(&dgraph_lock));
 974         assert(v->gv_type == GVT_GROUP);
 975         assert(uu_list_numnodes(v->gv_dependents) == 0);
 976 
 977         while ((e = uu_list_first(v->gv_dependencies)) != NULL) {
 978                 dv = e->ge_vertex;
 979 
 980                 graph_remove_edge(v, dv);
 981 
 982                 switch (dv->gv_type) {
 983                 case GVT_INST:          /* instance dependency */
 984                 case GVT_SVC:           /* service dependency */
 985                         (void) free_if_unrefed(dv);
 986                         break;
 987 
 988                 case GVT_FILE:          /* file dependency */
 989                         assert(uu_list_numnodes(dv->gv_dependencies) == 0);
 990                         if (uu_list_numnodes(dv->gv_dependents) == 0)
 991                                 graph_remove_vertex(dv);
 992                         break;
 993 
 994                 default:
 995 #ifndef NDEBUG
 996                         uu_warn("%s:%d: Unexpected node type %d", __FILE__,
 997                             __LINE__, dv->gv_type);
 998 #endif
 999                         abort();
1000                 }
1001         }
1002 
1003         graph_remove_vertex(v);
1004 }
1005 
1006 static int
1007 delete_instance_deps_cb(graph_edge_t *e, void **ptrs)
1008 {
1009         graph_vertex_t *v = ptrs[0];
1010         boolean_t delete_restarter_dep = (boolean_t)ptrs[1];
1011         graph_vertex_t *dv;
1012 
1013         dv = e->ge_vertex;
1014 
1015         /*
1016          * We have four possibilities here:
1017          *   - GVT_INST: restarter
1018          *   - GVT_GROUP - GVT_INST: instance dependency
1019          *   - GVT_GROUP - GVT_SVC - GV_INST: service dependency
1020          *   - GVT_GROUP - GVT_FILE: file dependency
1021          */
1022         switch (dv->gv_type) {
1023         case GVT_INST:  /* restarter */
1024                 assert(dv->gv_id == v->gv_restarter_id);
1025                 if (delete_restarter_dep)
1026                         graph_remove_edge(v, dv);
1027                 break;
1028 
1029         case GVT_GROUP: /* pg dependency */
1030                 graph_remove_edge(v, dv);
1031                 delete_depgroup(dv);
1032                 break;
1033 
1034         case GVT_FILE:
1035                 /* These are currently not direct dependencies */
1036 
1037         default:
1038 #ifndef NDEBUG
1039                 uu_warn("%s:%d: Bad vertex type %d.\n", __FILE__, __LINE__,
1040                     dv->gv_type);
1041 #endif
1042                 abort();
1043         }
1044 
1045         return (UU_WALK_NEXT);
1046 }
1047 
1048 static void
1049 delete_instance_dependencies(graph_vertex_t *v, boolean_t delete_restarter_dep)
1050 {
1051         void *ptrs[2];
1052         int r;
1053 
1054         assert(MUTEX_HELD(&dgraph_lock));
1055         assert(v->gv_type == GVT_INST);
1056 
1057         ptrs[0] = v;
1058         ptrs[1] = (void *)delete_restarter_dep;
1059 
1060         r = uu_list_walk(v->gv_dependencies,
1061             (uu_walk_fn_t *)delete_instance_deps_cb, &ptrs, UU_WALK_ROBUST);
1062         assert(r == 0);
1063 }
1064 
1065 /*
1066  * int graph_insert_vertex_unconfigured()
1067  *   Insert a vertex without sending any restarter events. If the vertex
1068  *   already exists or creation is successful, return a pointer to it in *vp.
1069  *
1070  *   If type is not GVT_GROUP, dt can remain unset.
1071  *
1072  *   Returns 0, EEXIST, or EINVAL if the arguments are invalid (i.e., fmri
1073  *   doesn't agree with type, or type doesn't agree with dt).
1074  */
1075 static int
1076 graph_insert_vertex_unconfigured(const char *fmri, gv_type_t type,
1077     depgroup_type_t dt, restarter_error_t rt, graph_vertex_t **vp)
1078 {
1079         int r;
1080         int i;
1081 
1082         assert(MUTEX_HELD(&dgraph_lock));
1083 
1084         switch (type) {
1085         case GVT_SVC:
1086         case GVT_INST:
1087                 if (strncmp(fmri, "svc:", sizeof ("svc:") - 1) != 0)
1088                         return (EINVAL);
1089                 break;
1090 
1091         case GVT_FILE:
1092                 if (strncmp(fmri, "file:", sizeof ("file:") - 1) != 0)
1093                         return (EINVAL);
1094                 break;
1095 
1096         case GVT_GROUP:
1097                 if (dt <= 0 || rt < 0)
1098                         return (EINVAL);
1099                 break;
1100 
1101         default:
1102 #ifndef NDEBUG
1103                 uu_warn("%s:%d: Unknown type %d.\n", __FILE__, __LINE__, type);
1104 #endif
1105                 abort();
1106         }
1107 
1108         *vp = vertex_get_by_name(fmri);
1109         if (*vp != NULL)
1110                 return (EEXIST);
1111 
1112         *vp = graph_add_vertex(fmri);
1113 
1114         (*vp)->gv_type = type;
1115         (*vp)->gv_depgroup = dt;
1116         (*vp)->gv_restart = rt;
1117 
1118         (*vp)->gv_flags = 0;
1119         (*vp)->gv_state = RESTARTER_STATE_NONE;
1120 
1121         for (i = 0; special_vertices[i].name != NULL; ++i) {
1122                 if (strcmp(fmri, special_vertices[i].name) == 0) {
1123                         (*vp)->gv_start_f = special_vertices[i].start_f;
1124                         (*vp)->gv_post_online_f =
1125                             special_vertices[i].post_online_f;
1126                         (*vp)->gv_post_disable_f =
1127                             special_vertices[i].post_disable_f;
1128                         break;
1129                 }
1130         }
1131 
1132         (*vp)->gv_restarter_id = -1;
1133         (*vp)->gv_restarter_channel = 0;
1134 
1135         if (type == GVT_INST) {
1136                 char *sfmri;
1137                 graph_vertex_t *sv;
1138 
1139                 sfmri = inst_fmri_to_svc_fmri(fmri);
1140                 sv = vertex_get_by_name(sfmri);
1141                 if (sv == NULL) {
1142                         r = graph_insert_vertex_unconfigured(sfmri, GVT_SVC, 0,
1143                             0, &sv);
1144                         assert(r == 0);
1145                 }
1146                 startd_free(sfmri, max_scf_fmri_size);
1147 
1148                 graph_add_edge(sv, *vp);
1149         }
1150 
1151         /*
1152          * If this vertex is in the subgraph, mark it as so, for both
1153          * GVT_INST and GVT_SERVICE verteces.
1154          * A GVT_SERVICE vertex can only be in the subgraph if another instance
1155          * depends on it, in which case it's already been added to the graph
1156          * and marked as in the subgraph (by refresh_vertex()).  If a
1157          * GVT_SERVICE vertex was freshly added (by the code above), it means
1158          * that it has no dependents, and cannot be in the subgraph.
1159          * Regardless of this, we still check that gv_flags includes
1160          * GV_INSUBGRAPH in the event that future behavior causes the above
1161          * code to add a GVT_SERVICE vertex which should be in the subgraph.
1162          */
1163 
1164         (*vp)->gv_flags |= (should_be_in_subgraph(*vp)? GV_INSUBGRAPH : 0);
1165 
1166         return (0);
1167 }
1168 
1169 /*
1170  * Returns 0 on success or ELOOP if the dependency would create a cycle.
1171  */
1172 static int
1173 graph_insert_dependency(graph_vertex_t *fv, graph_vertex_t *tv, int **pathp)
1174 {
1175         hrtime_t now;
1176 
1177         assert(MUTEX_HELD(&dgraph_lock));
1178 
1179         /* cycle detection */
1180         now = gethrtime();
1181 
1182         /* Don't follow exclusions. */
1183         if (!(fv->gv_type == GVT_GROUP &&
1184             fv->gv_depgroup == DEPGRP_EXCLUDE_ALL)) {
1185                 *pathp = is_path_to(tv, fv);
1186                 if (*pathp)
1187                         return (ELOOP);
1188         }
1189 
1190         dep_cycle_ns += gethrtime() - now;
1191         ++dep_inserts;
1192         now = gethrtime();
1193 
1194         graph_add_edge(fv, tv);
1195 
1196         dep_insert_ns += gethrtime() - now;
1197 
1198         /* Check if the dependency adds the "to" vertex to the subgraph */
1199         tv->gv_flags |= (should_be_in_subgraph(tv) ? GV_INSUBGRAPH : 0);
1200 
1201         return (0);
1202 }
1203 
1204 static int
1205 inst_running(graph_vertex_t *v)
1206 {
1207         assert(v->gv_type == GVT_INST);
1208 
1209         if (v->gv_state == RESTARTER_STATE_ONLINE ||
1210             v->gv_state == RESTARTER_STATE_DEGRADED)
1211                 return (1);
1212 
1213         return (0);
1214 }
1215 
1216 /*
1217  * The dependency evaluation functions return
1218  *   1 - dependency satisfied
1219  *   0 - dependency unsatisfied
1220  *   -1 - dependency unsatisfiable (without administrator intervention)
1221  *
1222  * The functions also take a boolean satbility argument.  When true, the
1223  * functions may recurse in order to determine satisfiability.
1224  */
1225 static int require_any_satisfied(graph_vertex_t *, boolean_t);
1226 static int dependency_satisfied(graph_vertex_t *, boolean_t);
1227 
1228 /*
1229  * A require_all dependency is unsatisfied if any elements are unsatisfied.  It
1230  * is unsatisfiable if any elements are unsatisfiable.
1231  */
1232 static int
1233 require_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1234 {
1235         graph_edge_t *edge;
1236         int i;
1237         boolean_t any_unsatisfied;
1238 
1239         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1240                 return (1);
1241 
1242         any_unsatisfied = B_FALSE;
1243 
1244         for (edge = uu_list_first(groupv->gv_dependencies);
1245             edge != NULL;
1246             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1247                 i = dependency_satisfied(edge->ge_vertex, satbility);
1248                 if (i == 1)
1249                         continue;
1250 
1251                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1252                     "require_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1253                     edge->ge_vertex->gv_name, i == 0 ? "ed" : "able");
1254 
1255                 if (!satbility)
1256                         return (0);
1257 
1258                 if (i == -1)
1259                         return (-1);
1260 
1261                 any_unsatisfied = B_TRUE;
1262         }
1263 
1264         return (any_unsatisfied ? 0 : 1);
1265 }
1266 
1267 /*
1268  * A require_any dependency is satisfied if any element is satisfied.  It is
1269  * satisfiable if any element is satisfiable.
1270  */
1271 static int
1272 require_any_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1273 {
1274         graph_edge_t *edge;
1275         int s;
1276         boolean_t satisfiable;
1277 
1278         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1279                 return (1);
1280 
1281         satisfiable = B_FALSE;
1282 
1283         for (edge = uu_list_first(groupv->gv_dependencies);
1284             edge != NULL;
1285             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1286                 s = dependency_satisfied(edge->ge_vertex, satbility);
1287 
1288                 if (s == 1)
1289                         return (1);
1290 
1291                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1292                     "require_any(%s): %s is unsatisfi%s.\n",
1293                     groupv->gv_name, edge->ge_vertex->gv_name,
1294                     s == 0 ? "ed" : "able");
1295 
1296                 if (satbility && s == 0)
1297                         satisfiable = B_TRUE;
1298         }
1299 
1300         return (!satbility || satisfiable ? 0 : -1);
1301 }
1302 
1303 /*
1304  * An optional_all dependency only considers elements which are configured,
1305  * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1306  * is unsatisfied.
1307  *
1308  * Offline dependencies which are waiting for a dependency to come online are
1309  * unsatisfied.  Offline dependences which cannot possibly come online
1310  * (unsatisfiable) are always considered satisfied.
1311  */
1312 static int
1313 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1314 {
1315         graph_edge_t *edge;
1316         graph_vertex_t *v;
1317         boolean_t any_qualified;
1318         boolean_t any_unsatisfied;
1319         int i;
1320 
1321         any_qualified = B_FALSE;
1322         any_unsatisfied = B_FALSE;
1323 
1324         for (edge = uu_list_first(groupv->gv_dependencies);
1325             edge != NULL;
1326             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1327                 v = edge->ge_vertex;
1328 
1329                 switch (v->gv_type) {
1330                 case GVT_INST:
1331                         /* Skip missing instances */
1332                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1333                                 continue;
1334 
1335                         if (v->gv_state == RESTARTER_STATE_MAINT)
1336                                 continue;
1337 
1338                         any_qualified = B_TRUE;
1339                         if (v->gv_state == RESTARTER_STATE_OFFLINE) {
1340                                 /*
1341                                  * For offline dependencies, treat unsatisfiable
1342                                  * as satisfied.
1343                                  */
1344                                 i = dependency_satisfied(v, B_TRUE);
1345                                 if (i == -1)
1346                                         i = 1;
1347                         } else if (v->gv_state == RESTARTER_STATE_DISABLED) {
1348                                 /*
1349                                  * If the instance is transitioning out of
1350                                  * disabled the dependency is temporarily
1351                                  * unsatisfied (not unsatisfiable).
1352                                  */
1353                                 i = v->gv_flags & GV_ENABLED ? 0 : 1;
1354                         } else {
1355                                 i = dependency_satisfied(v, satbility);
1356                         }
1357                         break;
1358 
1359                 case GVT_FILE:
1360                         any_qualified = B_TRUE;
1361                         i = dependency_satisfied(v, satbility);
1362 
1363                         break;
1364 
1365                 case GVT_SVC: {
1366                         any_qualified = B_TRUE;
1367                         i = optional_all_satisfied(v, satbility);
1368 
1369                         break;
1370                 }
1371 
1372                 case GVT_GROUP:
1373                 default:
1374 #ifndef NDEBUG
1375                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1376                             __LINE__, v->gv_type);
1377 #endif
1378                         abort();
1379                 }
1380 
1381                 if (i == 1)
1382                         continue;
1383 
1384                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1385                     "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1386                     v->gv_name, i == 0 ? "ed" : "able");
1387 
1388                 if (!satbility)
1389                         return (0);
1390                 if (i == -1)
1391                         return (-1);
1392                 any_unsatisfied = B_TRUE;
1393         }
1394 
1395         if (!any_qualified)
1396                 return (1);
1397 
1398         return (any_unsatisfied ? 0 : 1);
1399 }
1400 
1401 /*
1402  * An exclude_all dependency is unsatisfied if any non-service element is
1403  * satisfied or any service instance which is configured, enabled, and not in
1404  * maintenance is satisfied.  Usually when unsatisfied, it is also
1405  * unsatisfiable.
1406  */
1407 #define LOG_EXCLUDE(u, v)                                               \
1408         log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,                   \
1409             "exclude_all(%s): %s is satisfied.\n",                      \
1410             (u)->gv_name, (v)->gv_name)
1411 
1412 /* ARGSUSED */
1413 static int
1414 exclude_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1415 {
1416         graph_edge_t *edge, *e2;
1417         graph_vertex_t *v, *v2;
1418 
1419         for (edge = uu_list_first(groupv->gv_dependencies);
1420             edge != NULL;
1421             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1422                 v = edge->ge_vertex;
1423 
1424                 switch (v->gv_type) {
1425                 case GVT_INST:
1426                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1427                                 continue;
1428 
1429                         switch (v->gv_state) {
1430                         case RESTARTER_STATE_ONLINE:
1431                         case RESTARTER_STATE_DEGRADED:
1432                                 LOG_EXCLUDE(groupv, v);
1433                                 return (v->gv_flags & GV_ENABLED ? -1 : 0);
1434 
1435                         case RESTARTER_STATE_OFFLINE:
1436                         case RESTARTER_STATE_UNINIT:
1437                                 LOG_EXCLUDE(groupv, v);
1438                                 return (0);
1439 
1440                         case RESTARTER_STATE_DISABLED:
1441                         case RESTARTER_STATE_MAINT:
1442                                 continue;
1443 
1444                         default:
1445 #ifndef NDEBUG
1446                                 uu_warn("%s:%d: Unexpected vertex state %d.\n",
1447                                     __FILE__, __LINE__, v->gv_state);
1448 #endif
1449                                 abort();
1450                         }
1451                         /* NOTREACHED */
1452 
1453                 case GVT_SVC:
1454                         break;
1455 
1456                 case GVT_FILE:
1457                         if (!file_ready(v))
1458                                 continue;
1459                         LOG_EXCLUDE(groupv, v);
1460                         return (-1);
1461 
1462                 case GVT_GROUP:
1463                 default:
1464 #ifndef NDEBUG
1465                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1466                             __LINE__, v->gv_type);
1467 #endif
1468                         abort();
1469                 }
1470 
1471                 /* v represents a service */
1472                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1473                         continue;
1474 
1475                 for (e2 = uu_list_first(v->gv_dependencies);
1476                     e2 != NULL;
1477                     e2 = uu_list_next(v->gv_dependencies, e2)) {
1478                         v2 = e2->ge_vertex;
1479                         assert(v2->gv_type == GVT_INST);
1480 
1481                         if ((v2->gv_flags & GV_CONFIGURED) == 0)
1482                                 continue;
1483 
1484                         switch (v2->gv_state) {
1485                         case RESTARTER_STATE_ONLINE:
1486                         case RESTARTER_STATE_DEGRADED:
1487                                 LOG_EXCLUDE(groupv, v2);
1488                                 return (v2->gv_flags & GV_ENABLED ? -1 : 0);
1489 
1490                         case RESTARTER_STATE_OFFLINE:
1491                         case RESTARTER_STATE_UNINIT:
1492                                 LOG_EXCLUDE(groupv, v2);
1493                                 return (0);
1494 
1495                         case RESTARTER_STATE_DISABLED:
1496                         case RESTARTER_STATE_MAINT:
1497                                 continue;
1498 
1499                         default:
1500 #ifndef NDEBUG
1501                                 uu_warn("%s:%d: Unexpected vertex type %d.\n",
1502                                     __FILE__, __LINE__, v2->gv_type);
1503 #endif
1504                                 abort();
1505                         }
1506                 }
1507         }
1508 
1509         return (1);
1510 }
1511 
1512 /*
1513  * int instance_satisfied()
1514  *   Determine if all the dependencies are satisfied for the supplied instance
1515  *   vertex. Return 1 if they are, 0 if they aren't, and -1 if they won't be
1516  *   without administrator intervention.
1517  */
1518 static int
1519 instance_satisfied(graph_vertex_t *v, boolean_t satbility)
1520 {
1521         assert(v->gv_type == GVT_INST);
1522         assert(!inst_running(v));
1523 
1524         return (require_all_satisfied(v, satbility));
1525 }
1526 
1527 /*
1528  * Decide whether v can satisfy a dependency.  v can either be a child of
1529  * a group vertex, or of an instance vertex.
1530  */
1531 static int
1532 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1533 {
1534         switch (v->gv_type) {
1535         case GVT_INST:
1536                 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1537                         if (v->gv_flags & GV_DEATHROW) {
1538                                 /*
1539                                  * A dependency on an instance with GV_DEATHROW
1540                                  * flag is always considered as satisfied.
1541                                  */
1542                                 return (1);
1543                         }
1544                         return (-1);
1545                 }
1546 
1547                 /*
1548                  * Any vertex with the GV_TODISABLE flag set is guaranteed
1549                  * to have its dependencies unsatisfiable.  Any vertex with
1550                  * GV_TOOFFLINE may be satisfied after it transitions.
1551                  */
1552                 if (v->gv_flags & GV_TODISABLE)
1553                         return (-1);
1554                 if (v->gv_flags & GV_TOOFFLINE)
1555                         return (0);
1556 
1557                 switch (v->gv_state) {
1558                 case RESTARTER_STATE_ONLINE:
1559                 case RESTARTER_STATE_DEGRADED:
1560                         return (1);
1561 
1562                 case RESTARTER_STATE_OFFLINE:
1563                         if (!satbility)
1564                                 return (0);
1565                         return (instance_satisfied(v, satbility) != -1 ?
1566                             0 : -1);
1567 
1568                 case RESTARTER_STATE_DISABLED:
1569                 case RESTARTER_STATE_MAINT:
1570                         return (-1);
1571 
1572                 case RESTARTER_STATE_UNINIT:
1573                         return (0);
1574 
1575                 default:
1576 #ifndef NDEBUG
1577                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
1578                             __FILE__, __LINE__, v->gv_state);
1579 #endif
1580                         abort();
1581                         /* NOTREACHED */
1582                 }
1583 
1584         case GVT_SVC:
1585                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1586                         return (-1);
1587                 return (require_any_satisfied(v, satbility));
1588 
1589         case GVT_FILE:
1590                 /* i.e., we assume files will not be automatically generated */
1591                 return (file_ready(v) ? 1 : -1);
1592 
1593         case GVT_GROUP:
1594                 break;
1595 
1596         default:
1597 #ifndef NDEBUG
1598                 uu_warn("%s:%d: Unexpected node type %d.\n", __FILE__, __LINE__,
1599                     v->gv_type);
1600 #endif
1601                 abort();
1602                 /* NOTREACHED */
1603         }
1604 
1605         switch (v->gv_depgroup) {
1606         case DEPGRP_REQUIRE_ANY:
1607                 return (require_any_satisfied(v, satbility));
1608 
1609         case DEPGRP_REQUIRE_ALL:
1610                 return (require_all_satisfied(v, satbility));
1611 
1612         case DEPGRP_OPTIONAL_ALL:
1613                 return (optional_all_satisfied(v, satbility));
1614 
1615         case DEPGRP_EXCLUDE_ALL:
1616                 return (exclude_all_satisfied(v, satbility));
1617 
1618         default:
1619 #ifndef NDEBUG
1620                 uu_warn("%s:%d: Unknown dependency grouping %d.\n", __FILE__,
1621                     __LINE__, v->gv_depgroup);
1622 #endif
1623                 abort();
1624         }
1625 }
1626 
1627 void
1628 graph_start_if_satisfied(graph_vertex_t *v)
1629 {
1630         if (v->gv_state == RESTARTER_STATE_OFFLINE &&
1631             instance_satisfied(v, B_FALSE) == 1) {
1632                 if (v->gv_start_f == NULL)
1633                         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
1634                 else
1635                         v->gv_start_f(v);
1636         }
1637 }
1638 
1639 /*
1640  * propagate_satbility()
1641  *
1642  * This function is used when the given vertex changes state in such a way that
1643  * one of its dependents may become unsatisfiable.  This happens when an
1644  * instance transitions between offline -> online, or from !running ->
1645  * maintenance, as well as when an instance is removed from the graph.
1646  *
1647  * We have to walk all the dependents, since optional_all dependencies several
1648  * levels up could become (un)satisfied, instead of unsatisfiable.  For example,
1649  *
1650  *      +-----+  optional_all  +-----+  require_all  +-----+
1651  *      |  A  |--------------->|  B  |-------------->|  C  |
1652  *      +-----+                +-----+               +-----+
1653  *
1654  *                                              offline -> maintenance
1655  *
1656  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1657  * an optional dependency, what was previously an unsatisfiable situation is now
1658  * satisfied (B will never come online, even though its state hasn't changed).
1659  *
1660  * Note that it's not necessary to continue examining dependents after reaching
1661  * an optional_all dependency.  It's not possible for an optional_all dependency
1662  * to change satisfiability without also coming online, in which case we get a
1663  * start event and propagation continues naturally.  However, it does no harm to
1664  * continue propagating satisfiability (as it is a relatively rare event), and
1665  * keeps the walker code simple and generic.
1666  */
1667 /*ARGSUSED*/
1668 static int
1669 satbility_cb(graph_vertex_t *v, void *arg)
1670 {
1671         if (v->gv_flags & GV_TOOFFLINE)
1672                 return (UU_WALK_NEXT);
1673 
1674         if (v->gv_type == GVT_INST)
1675                 graph_start_if_satisfied(v);
1676 
1677         return (UU_WALK_NEXT);
1678 }
1679 
1680 static void
1681 propagate_satbility(graph_vertex_t *v)
1682 {
1683         graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1684 }
1685 
1686 static void propagate_stop(graph_vertex_t *, void *);
1687 
1688 /*
1689  * propagate_start()
1690  *
1691  * This function is used to propagate a start event to the dependents of the
1692  * given vertex.  Any dependents that are offline but have their dependencies
1693  * satisfied are started.  Any dependents that are online and have restart_on
1694  * set to "restart" or "refresh" are restarted because their dependencies have
1695  * just changed.  This only happens with optional_all dependencies.
1696  */
1697 static void
1698 propagate_start(graph_vertex_t *v, void *arg)
1699 {
1700         restarter_error_t err = (restarter_error_t)arg;
1701 
1702         if (v->gv_flags & GV_TOOFFLINE)
1703                 return;
1704 
1705         switch (v->gv_type) {
1706         case GVT_INST:
1707                 /* Restarter */
1708                 if (inst_running(v)) {
1709                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1710                                 vertex_send_event(v,
1711                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1712                         }
1713                 } else {
1714                         graph_start_if_satisfied(v);
1715                 }
1716                 break;
1717 
1718         case GVT_GROUP:
1719                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1720                         graph_walk_dependents(v, propagate_stop,
1721                             (void *)RERR_RESTART);
1722                         break;
1723                 }
1724                 err = v->gv_restart;
1725                 /* FALLTHROUGH */
1726 
1727         case GVT_SVC:
1728                 graph_walk_dependents(v, propagate_start, (void *)err);
1729                 break;
1730 
1731         case GVT_FILE:
1732 #ifndef NDEBUG
1733                 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1734                     __FILE__, __LINE__);
1735 #endif
1736                 abort();
1737                 /* NOTREACHED */
1738 
1739         default:
1740 #ifndef NDEBUG
1741                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1742                     v->gv_type);
1743 #endif
1744                 abort();
1745         }
1746 }
1747 
1748 /*
1749  * propagate_stop()
1750  *
1751  * This function is used to propagate a stop event to the dependents of the
1752  * given vertex.  Any dependents that are online (or in degraded state) with
1753  * the restart_on property set to "restart" or "refresh" will be stopped as
1754  * their dependencies have just changed, propagate_start() will start them
1755  * again once their dependencies have been re-satisfied.
1756  */
1757 static void
1758 propagate_stop(graph_vertex_t *v, void *arg)
1759 {
1760         restarter_error_t err = (restarter_error_t)arg;
1761 
1762         if (v->gv_flags & GV_TOOFFLINE)
1763                 return;
1764 
1765         switch (v->gv_type) {
1766         case GVT_INST:
1767                 /* Restarter */
1768                 if (err > RERR_NONE && inst_running(v)) {
1769                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1770                                 vertex_send_event(v,
1771                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1772                         } else {
1773                                 vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1774                         }
1775                 }
1776                 break;
1777 
1778         case GVT_SVC:
1779                 graph_walk_dependents(v, propagate_stop, arg);
1780                 break;
1781 
1782         case GVT_FILE:
1783 #ifndef NDEBUG
1784                 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1785                     __FILE__, __LINE__);
1786 #endif
1787                 abort();
1788                 /* NOTREACHED */
1789 
1790         case GVT_GROUP:
1791                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1792                         graph_walk_dependents(v, propagate_start, NULL);
1793                         break;
1794                 }
1795 
1796                 if (err == RERR_NONE || err > v->gv_restart)
1797                         break;
1798 
1799                 graph_walk_dependents(v, propagate_stop, arg);
1800                 break;
1801 
1802         default:
1803 #ifndef NDEBUG
1804                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1805                     v->gv_type);
1806 #endif
1807                 abort();
1808         }
1809 }
1810 
1811 void
1812 offline_vertex(graph_vertex_t *v)
1813 {
1814         scf_handle_t *h = libscf_handle_create_bound_loop();
1815         scf_instance_t *scf_inst = safe_scf_instance_create(h);
1816         scf_propertygroup_t *pg = safe_scf_pg_create(h);
1817         restarter_instance_state_t state, next_state;
1818         int r;
1819 
1820         assert(v->gv_type == GVT_INST);
1821 
1822         if (scf_inst == NULL)
1823                 bad_error("safe_scf_instance_create", scf_error());
1824         if (pg == NULL)
1825                 bad_error("safe_scf_pg_create", scf_error());
1826 
1827         /* if the vertex is already going offline, return */
1828 rep_retry:
1829         if (scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, scf_inst, NULL,
1830             NULL, SCF_DECODE_FMRI_EXACT) != 0) {
1831                 switch (scf_error()) {
1832                 case SCF_ERROR_CONNECTION_BROKEN:
1833                         libscf_handle_rebind(h);
1834                         goto rep_retry;
1835 
1836                 case SCF_ERROR_NOT_FOUND:
1837                         scf_pg_destroy(pg);
1838                         scf_instance_destroy(scf_inst);
1839                         (void) scf_handle_unbind(h);
1840                         scf_handle_destroy(h);
1841                         return;
1842                 }
1843                 uu_die("Can't decode FMRI %s: %s\n", v->gv_name,
1844                     scf_strerror(scf_error()));
1845         }
1846 
1847         r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
1848         if (r != 0) {
1849                 switch (scf_error()) {
1850                 case SCF_ERROR_CONNECTION_BROKEN:
1851                         libscf_handle_rebind(h);
1852                         goto rep_retry;
1853 
1854                 case SCF_ERROR_NOT_SET:
1855                 case SCF_ERROR_NOT_FOUND:
1856                         scf_pg_destroy(pg);
1857                         scf_instance_destroy(scf_inst);
1858                         (void) scf_handle_unbind(h);
1859                         scf_handle_destroy(h);
1860                         return;
1861 
1862                 default:
1863                         bad_error("scf_instance_get_pg", scf_error());
1864                 }
1865         } else {
1866                 r = libscf_read_states(pg, &state, &next_state);
1867                 if (r == 0 && (next_state == RESTARTER_STATE_OFFLINE ||
1868                     next_state == RESTARTER_STATE_DISABLED)) {
1869                         log_framework(LOG_DEBUG,
1870                             "%s: instance is already going down.\n",
1871                             v->gv_name);
1872                         scf_pg_destroy(pg);
1873                         scf_instance_destroy(scf_inst);
1874                         (void) scf_handle_unbind(h);
1875                         scf_handle_destroy(h);
1876                         return;
1877                 }
1878         }
1879 
1880         scf_pg_destroy(pg);
1881         scf_instance_destroy(scf_inst);
1882         (void) scf_handle_unbind(h);
1883         scf_handle_destroy(h);
1884 
1885         vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP_RESET);
1886 }
1887 
1888 /*
1889  * void graph_enable_by_vertex()
1890  *   If admin is non-zero, this is an administrative request for change
1891  *   of the enabled property.  Thus, send the ADMIN_DISABLE rather than
1892  *   a plain DISABLE restarter event.
1893  */
1894 void
1895 graph_enable_by_vertex(graph_vertex_t *vertex, int enable, int admin)
1896 {
1897         graph_vertex_t *v;
1898         int r;
1899 
1900         assert(MUTEX_HELD(&dgraph_lock));
1901         assert((vertex->gv_flags & GV_CONFIGURED));
1902 
1903         vertex->gv_flags = (vertex->gv_flags & ~GV_ENABLED) |
1904             (enable ? GV_ENABLED : 0);
1905 
1906         if (enable) {
1907                 if (vertex->gv_state != RESTARTER_STATE_OFFLINE &&
1908                     vertex->gv_state != RESTARTER_STATE_DEGRADED &&
1909                     vertex->gv_state != RESTARTER_STATE_ONLINE) {
1910                         /*
1911                          * In case the vertex was notified to go down,
1912                          * but now can return online, clear the _TOOFFLINE
1913                          * and _TODISABLE flags.
1914                          */
1915                         vertex->gv_flags &= ~GV_TOOFFLINE;
1916                         vertex->gv_flags &= ~GV_TODISABLE;
1917 
1918                         vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ENABLE);
1919                 }
1920 
1921                 /*
1922                  * Wait for state update from restarter before sending _START or
1923                  * _STOP.
1924                  */
1925 
1926                 return;
1927         }
1928 
1929         if (vertex->gv_state == RESTARTER_STATE_DISABLED)
1930                 return;
1931 
1932         if (!admin) {
1933                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_DISABLE);
1934 
1935                 /*
1936                  * Wait for state update from restarter before sending _START or
1937                  * _STOP.
1938                  */
1939 
1940                 return;
1941         }
1942 
1943         /*
1944          * If it is a DISABLE event requested by the administrator then we are
1945          * offlining the dependents first.
1946          */
1947 
1948         /*
1949          * Set GV_TOOFFLINE for the services we are offlining. We cannot
1950          * clear the GV_TOOFFLINE bits from all the services because
1951          * other DISABLE events might be handled at the same time.
1952          */
1953         vertex->gv_flags |= GV_TOOFFLINE;
1954 
1955         /* remember which vertex to disable... */
1956         vertex->gv_flags |= GV_TODISABLE;
1957 
1958         log_framework(LOG_DEBUG, "Marking in-subtree vertices before "
1959             "disabling %s.\n", vertex->gv_name);
1960 
1961         /* set GV_TOOFFLINE for its dependents */
1962         r = uu_list_walk(vertex->gv_dependents, (uu_walk_fn_t *)mark_subtree,
1963             NULL, 0);
1964         assert(r == 0);
1965 
1966         /* disable the instance now if there is nothing else to offline */
1967         if (insubtree_dependents_down(vertex) == B_TRUE) {
1968                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
1969                 return;
1970         }
1971 
1972         /*
1973          * This loop is similar to the one used for the graph reversal shutdown
1974          * and could be improved in term of performance for the subtree reversal
1975          * disable case.
1976          */
1977         for (v = uu_list_first(dgraph); v != NULL;
1978             v = uu_list_next(dgraph, v)) {
1979                 /* skip the vertex we are disabling for now */
1980                 if (v == vertex)
1981                         continue;
1982 
1983                 if (v->gv_type != GVT_INST ||
1984                     (v->gv_flags & GV_CONFIGURED) == 0 ||
1985                     (v->gv_flags & GV_ENABLED) == 0 ||
1986                     (v->gv_flags & GV_TOOFFLINE) == 0)
1987                         continue;
1988 
1989                 if ((v->gv_state != RESTARTER_STATE_ONLINE) &&
1990                     (v->gv_state != RESTARTER_STATE_DEGRADED)) {
1991                         /* continue if there is nothing to offline */
1992                         continue;
1993                 }
1994 
1995                 /*
1996                  * Instances which are up need to come down before we're
1997                  * done, but we can only offline the leaves here. An
1998                  * instance is a leaf when all its dependents are down.
1999                  */
2000                 if (insubtree_dependents_down(v) == B_TRUE) {
2001                         log_framework(LOG_DEBUG, "Offlining in-subtree "
2002                             "instance %s for %s.\n",
2003                             v->gv_name, vertex->gv_name);
2004                         offline_vertex(v);
2005                 }
2006         }
2007 }
2008 
2009 static int configure_vertex(graph_vertex_t *, scf_instance_t *);
2010 
2011 /*
2012  * Set the restarter for v to fmri_arg.  That is, make sure a vertex for
2013  * fmri_arg exists, make v depend on it, and send _ADD_INSTANCE for v.  If
2014  * v is already configured and fmri_arg indicates the current restarter, do
2015  * nothing.  If v is configured and fmri_arg is a new restarter, delete v's
2016  * dependency on the restarter, send _REMOVE_INSTANCE for v, and set the new
2017  * restarter.  Returns 0 on success, EINVAL if the FMRI is invalid,
2018  * ECONNABORTED if the repository connection is broken, and ELOOP
2019  * if the dependency would create a cycle.  In the last case, *pathp will
2020  * point to a -1-terminated array of ids which compose the path from v to
2021  * restarter_fmri.
2022  */
2023 int
2024 graph_change_restarter(graph_vertex_t *v, const char *fmri_arg, scf_handle_t *h,
2025     int **pathp)
2026 {
2027         char *restarter_fmri = NULL;
2028         graph_vertex_t *rv;
2029         int err;
2030         int id;
2031 
2032         assert(MUTEX_HELD(&dgraph_lock));
2033 
2034         if (fmri_arg[0] != '\0') {
2035                 err = fmri_canonify(fmri_arg, &restarter_fmri, B_TRUE);
2036                 if (err != 0) {
2037                         assert(err == EINVAL);
2038                         return (err);
2039                 }
2040         }
2041 
2042         if (restarter_fmri == NULL ||
2043             strcmp(restarter_fmri, SCF_SERVICE_STARTD) == 0) {
2044                 if (v->gv_flags & GV_CONFIGURED) {
2045                         if (v->gv_restarter_id == -1) {
2046                                 if (restarter_fmri != NULL)
2047                                         startd_free(restarter_fmri,
2048                                             max_scf_fmri_size);
2049                                 return (0);
2050                         }
2051 
2052                         graph_unset_restarter(v);
2053                 }
2054 
2055                 /* Master restarter, nothing to do. */
2056                 v->gv_restarter_id = -1;
2057                 v->gv_restarter_channel = NULL;
2058                 vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2059                 return (0);
2060         }
2061 
2062         if (v->gv_flags & GV_CONFIGURED) {
2063                 id = dict_lookup_byname(restarter_fmri);
2064                 if (id != -1 && v->gv_restarter_id == id) {
2065                         startd_free(restarter_fmri, max_scf_fmri_size);
2066                         return (0);
2067                 }
2068 
2069                 graph_unset_restarter(v);
2070         }
2071 
2072         err = graph_insert_vertex_unconfigured(restarter_fmri, GVT_INST, 0,
2073             RERR_NONE, &rv);
2074         startd_free(restarter_fmri, max_scf_fmri_size);
2075         assert(err == 0 || err == EEXIST);
2076 
2077         if (rv->gv_delegate_initialized == 0) {
2078                 if ((rv->gv_delegate_channel = restarter_protocol_init_delegate(
2079                     rv->gv_name)) == NULL)
2080                         return (EINVAL);
2081                 rv->gv_delegate_initialized = 1;
2082         }
2083         v->gv_restarter_id = rv->gv_id;
2084         v->gv_restarter_channel = rv->gv_delegate_channel;
2085 
2086         err = graph_insert_dependency(v, rv, pathp);
2087         if (err != 0) {
2088                 assert(err == ELOOP);
2089                 return (ELOOP);
2090         }
2091 
2092         vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2093 
2094         if (!(rv->gv_flags & GV_CONFIGURED)) {
2095                 scf_instance_t *inst;
2096 
2097                 err = libscf_fmri_get_instance(h, rv->gv_name, &inst);
2098                 switch (err) {
2099                 case 0:
2100                         err = configure_vertex(rv, inst);
2101                         scf_instance_destroy(inst);
2102                         switch (err) {
2103                         case 0:
2104                         case ECANCELED:
2105                                 break;
2106 
2107                         case ECONNABORTED:
2108                                 return (ECONNABORTED);
2109 
2110                         default:
2111                                 bad_error("configure_vertex", err);
2112                         }
2113                         break;
2114 
2115                 case ECONNABORTED:
2116                         return (ECONNABORTED);
2117 
2118                 case ENOENT:
2119                         break;
2120 
2121                 case ENOTSUP:
2122                         /*
2123                          * The fmri doesn't specify an instance - translate
2124                          * to EINVAL.
2125                          */
2126                         return (EINVAL);
2127 
2128                 case EINVAL:
2129                 default:
2130                         bad_error("libscf_fmri_get_instance", err);
2131                 }
2132         }
2133 
2134         return (0);
2135 }
2136 
2137 
2138 /*
2139  * Add all of the instances of the service named by fmri to the graph.
2140  * Returns
2141  *   0 - success
2142  *   ENOENT - service indicated by fmri does not exist
2143  *
2144  * In both cases *reboundp will be B_TRUE if the handle was rebound, or B_FALSE
2145  * otherwise.
2146  */
2147 static int
2148 add_service(const char *fmri, scf_handle_t *h, boolean_t *reboundp)
2149 {
2150         scf_service_t *svc;
2151         scf_instance_t *inst;
2152         scf_iter_t *iter;
2153         char *inst_fmri;
2154         int ret, r;
2155 
2156         *reboundp = B_FALSE;
2157 
2158         svc = safe_scf_service_create(h);
2159         inst = safe_scf_instance_create(h);
2160         iter = safe_scf_iter_create(h);
2161         inst_fmri = startd_alloc(max_scf_fmri_size);
2162 
2163 rebound:
2164         if (scf_handle_decode_fmri(h, fmri, NULL, svc, NULL, NULL, NULL,
2165             SCF_DECODE_FMRI_EXACT) != 0) {
2166                 switch (scf_error()) {
2167                 case SCF_ERROR_CONNECTION_BROKEN:
2168                 default:
2169                         libscf_handle_rebind(h);
2170                         *reboundp = B_TRUE;
2171                         goto rebound;
2172 
2173                 case SCF_ERROR_NOT_FOUND:
2174                         ret = ENOENT;
2175                         goto out;
2176 
2177                 case SCF_ERROR_INVALID_ARGUMENT:
2178                 case SCF_ERROR_CONSTRAINT_VIOLATED:
2179                 case SCF_ERROR_NOT_BOUND:
2180                 case SCF_ERROR_HANDLE_MISMATCH:
2181                         bad_error("scf_handle_decode_fmri", scf_error());
2182                 }
2183         }
2184 
2185         if (scf_iter_service_instances(iter, svc) != 0) {
2186                 switch (scf_error()) {
2187                 case SCF_ERROR_CONNECTION_BROKEN:
2188                 default:
2189                         libscf_handle_rebind(h);
2190                         *reboundp = B_TRUE;
2191                         goto rebound;
2192 
2193                 case SCF_ERROR_DELETED:
2194                         ret = ENOENT;
2195                         goto out;
2196 
2197                 case SCF_ERROR_HANDLE_MISMATCH:
2198                 case SCF_ERROR_NOT_BOUND:
2199                 case SCF_ERROR_NOT_SET:
2200                         bad_error("scf_iter_service_instances", scf_error());
2201                 }
2202         }
2203 
2204         for (;;) {
2205                 r = scf_iter_next_instance(iter, inst);
2206                 if (r == 0)
2207                         break;
2208                 if (r != 1) {
2209                         switch (scf_error()) {
2210                         case SCF_ERROR_CONNECTION_BROKEN:
2211                         default:
2212                                 libscf_handle_rebind(h);
2213                                 *reboundp = B_TRUE;
2214                                 goto rebound;
2215 
2216                         case SCF_ERROR_DELETED:
2217                                 ret = ENOENT;
2218                                 goto out;
2219 
2220                         case SCF_ERROR_HANDLE_MISMATCH:
2221                         case SCF_ERROR_NOT_BOUND:
2222                         case SCF_ERROR_NOT_SET:
2223                         case SCF_ERROR_INVALID_ARGUMENT:
2224                                 bad_error("scf_iter_next_instance",
2225                                     scf_error());
2226                         }
2227                 }
2228 
2229                 if (scf_instance_to_fmri(inst, inst_fmri, max_scf_fmri_size) <
2230                     0) {
2231                         switch (scf_error()) {
2232                         case SCF_ERROR_CONNECTION_BROKEN:
2233                                 libscf_handle_rebind(h);
2234                                 *reboundp = B_TRUE;
2235                                 goto rebound;
2236 
2237                         case SCF_ERROR_DELETED:
2238                                 continue;
2239 
2240                         case SCF_ERROR_NOT_BOUND:
2241                         case SCF_ERROR_NOT_SET:
2242                                 bad_error("scf_instance_to_fmri", scf_error());
2243                         }
2244                 }
2245 
2246                 r = dgraph_add_instance(inst_fmri, inst, B_FALSE);
2247                 switch (r) {
2248                 case 0:
2249                 case ECANCELED:
2250                         break;
2251 
2252                 case EEXIST:
2253                         continue;
2254 
2255                 case ECONNABORTED:
2256                         libscf_handle_rebind(h);
2257                         *reboundp = B_TRUE;
2258                         goto rebound;
2259 
2260                 case EINVAL:
2261                 default:
2262                         bad_error("dgraph_add_instance", r);
2263                 }
2264         }
2265 
2266         ret = 0;
2267 
2268 out:
2269         startd_free(inst_fmri, max_scf_fmri_size);
2270         scf_iter_destroy(iter);
2271         scf_instance_destroy(inst);
2272         scf_service_destroy(svc);
2273         return (ret);
2274 }
2275 
2276 struct depfmri_info {
2277         graph_vertex_t  *v;             /* GVT_GROUP vertex */
2278         gv_type_t       type;           /* type of dependency */
2279         const char      *inst_fmri;     /* FMRI of parental GVT_INST vert. */
2280         const char      *pg_name;       /* Name of dependency pg */
2281         scf_handle_t    *h;
2282         int             err;            /* return error code */
2283         int             **pathp;        /* return circular dependency path */
2284 };
2285 
2286 /*
2287  * Find or create a vertex for fmri and make info->v depend on it.
2288  * Returns
2289  *   0 - success
2290  *   nonzero - failure
2291  *
2292  * On failure, sets info->err to
2293  *   EINVAL - fmri is invalid
2294  *            fmri does not match info->type
2295  *   ELOOP - Adding the dependency creates a circular dependency.  *info->pathp
2296  *           will point to an array of the ids of the members of the cycle.
2297  *   ECONNABORTED - repository connection was broken
2298  *   ECONNRESET - succeeded, but repository connection was reset
2299  */
2300 static int
2301 process_dependency_fmri(const char *fmri, struct depfmri_info *info)
2302 {
2303         int err;
2304         graph_vertex_t *depgroup_v, *v;
2305         char *fmri_copy, *cfmri;
2306         size_t fmri_copy_sz;
2307         const char *scope, *service, *instance, *pg;
2308         scf_instance_t *inst;
2309         boolean_t rebound;
2310 
2311         assert(MUTEX_HELD(&dgraph_lock));
2312 
2313         /* Get or create vertex for FMRI */
2314         depgroup_v = info->v;
2315 
2316         if (strncmp(fmri, "file:", sizeof ("file:") - 1) == 0) {
2317                 if (info->type != GVT_FILE) {
2318                         log_framework(LOG_NOTICE,
2319                             "FMRI \"%s\" is not allowed for the \"%s\" "
2320                             "dependency's type of instance %s.\n", fmri,
2321                             info->pg_name, info->inst_fmri);
2322                         return (info->err = EINVAL);
2323                 }
2324 
2325                 err = graph_insert_vertex_unconfigured(fmri, info->type, 0,
2326                     RERR_NONE, &v);
2327                 switch (err) {
2328                 case 0:
2329                         break;
2330 
2331                 case EEXIST:
2332                         assert(v->gv_type == GVT_FILE);
2333                         break;
2334 
2335                 case EINVAL:            /* prevented above */
2336                 default:
2337                         bad_error("graph_insert_vertex_unconfigured", err);
2338                 }
2339         } else {
2340                 if (info->type != GVT_INST) {
2341                         log_framework(LOG_NOTICE,
2342                             "FMRI \"%s\" is not allowed for the \"%s\" "
2343                             "dependency's type of instance %s.\n", fmri,
2344                             info->pg_name, info->inst_fmri);
2345                         return (info->err = EINVAL);
2346                 }
2347 
2348                 /*
2349                  * We must canonify fmri & add a vertex for it.
2350                  */
2351                 fmri_copy_sz = strlen(fmri) + 1;
2352                 fmri_copy = startd_alloc(fmri_copy_sz);
2353                 (void) strcpy(fmri_copy, fmri);
2354 
2355                 /* Determine if the FMRI is a property group or instance */
2356                 if (scf_parse_svc_fmri(fmri_copy, &scope, &service,
2357                     &instance, &pg, NULL) != 0) {
2358                         startd_free(fmri_copy, fmri_copy_sz);
2359                         log_framework(LOG_NOTICE,
2360                             "Dependency \"%s\" of %s has invalid FMRI "
2361                             "\"%s\".\n", info->pg_name, info->inst_fmri,
2362                             fmri);
2363                         return (info->err = EINVAL);
2364                 }
2365 
2366                 if (service == NULL || pg != NULL) {
2367                         startd_free(fmri_copy, fmri_copy_sz);
2368                         log_framework(LOG_NOTICE,
2369                             "Dependency \"%s\" of %s does not designate a "
2370                             "service or instance.\n", info->pg_name,
2371                             info->inst_fmri);
2372                         return (info->err = EINVAL);
2373                 }
2374 
2375                 if (scope == NULL || strcmp(scope, SCF_SCOPE_LOCAL) == 0) {
2376                         cfmri = uu_msprintf("svc:/%s%s%s",
2377                             service, instance ? ":" : "", instance ? instance :
2378                             "");
2379                 } else {
2380                         cfmri = uu_msprintf("svc://%s/%s%s%s",
2381                             scope, service, instance ? ":" : "", instance ?
2382                             instance : "");
2383                 }
2384 
2385                 startd_free(fmri_copy, fmri_copy_sz);
2386 
2387                 err = graph_insert_vertex_unconfigured(cfmri, instance ?
2388                     GVT_INST : GVT_SVC, instance ? 0 : DEPGRP_REQUIRE_ANY,
2389                     RERR_NONE, &v);
2390                 uu_free(cfmri);
2391                 switch (err) {
2392                 case 0:
2393                         break;
2394 
2395                 case EEXIST:
2396                         /* Verify v. */
2397                         if (instance != NULL)
2398                                 assert(v->gv_type == GVT_INST);
2399                         else
2400                                 assert(v->gv_type == GVT_SVC);
2401                         break;
2402 
2403                 default:
2404                         bad_error("graph_insert_vertex_unconfigured", err);
2405                 }
2406         }
2407 
2408         /* Add dependency from depgroup_v to new vertex */
2409         info->err = graph_insert_dependency(depgroup_v, v, info->pathp);
2410         switch (info->err) {
2411         case 0:
2412                 break;
2413 
2414         case ELOOP:
2415                 return (ELOOP);
2416 
2417         default:
2418                 bad_error("graph_insert_dependency", info->err);
2419         }
2420 
2421         /* This must be after we insert the dependency, to avoid looping. */
2422         switch (v->gv_type) {
2423         case GVT_INST:
2424                 if ((v->gv_flags & GV_CONFIGURED) != 0)
2425                         break;
2426 
2427                 inst = safe_scf_instance_create(info->h);
2428 
2429                 rebound = B_FALSE;
2430 
2431 rebound:
2432                 err = libscf_lookup_instance(v->gv_name, inst);
2433                 switch (err) {
2434                 case 0:
2435                         err = configure_vertex(v, inst);
2436                         switch (err) {
2437                         case 0:
2438                         case ECANCELED:
2439                                 break;
2440 
2441                         case ECONNABORTED:
2442                                 libscf_handle_rebind(info->h);
2443                                 rebound = B_TRUE;
2444                                 goto rebound;
2445 
2446                         default:
2447                                 bad_error("configure_vertex", err);
2448                         }
2449                         break;
2450 
2451                 case ENOENT:
2452                         break;
2453 
2454                 case ECONNABORTED:
2455                         libscf_handle_rebind(info->h);
2456                         rebound = B_TRUE;
2457                         goto rebound;
2458 
2459                 case EINVAL:
2460                 case ENOTSUP:
2461                 default:
2462                         bad_error("libscf_fmri_get_instance", err);
2463                 }
2464 
2465                 scf_instance_destroy(inst);
2466 
2467                 if (rebound)
2468                         return (info->err = ECONNRESET);
2469                 break;
2470 
2471         case GVT_SVC:
2472                 (void) add_service(v->gv_name, info->h, &rebound);
2473                 if (rebound)
2474                         return (info->err = ECONNRESET);
2475         }
2476 
2477         return (0);
2478 }
2479 
2480 struct deppg_info {
2481         graph_vertex_t  *v;             /* GVT_INST vertex */
2482         int             err;            /* return error */
2483         int             **pathp;        /* return circular dependency path */
2484 };
2485 
2486 /*
2487  * Make info->v depend on a new GVT_GROUP node for this property group,
2488  * and then call process_dependency_fmri() for the values of the entity
2489  * property.  Return 0 on success, or if something goes wrong return nonzero
2490  * and set info->err to ECONNABORTED, EINVAL, or the error code returned by
2491  * process_dependency_fmri().
2492  */
2493 static int
2494 process_dependency_pg(scf_propertygroup_t *pg, struct deppg_info *info)
2495 {
2496         scf_handle_t *h;
2497         depgroup_type_t deptype;
2498         restarter_error_t rerr;
2499         struct depfmri_info linfo;
2500         char *fmri, *pg_name;
2501         size_t fmri_sz;
2502         graph_vertex_t *depgrp;
2503         scf_property_t *prop;
2504         int err;
2505         int empty;
2506         scf_error_t scferr;
2507         ssize_t len;
2508 
2509         assert(MUTEX_HELD(&dgraph_lock));
2510 
2511         h = scf_pg_handle(pg);
2512 
2513         pg_name = startd_alloc(max_scf_name_size);
2514 
2515         len = scf_pg_get_name(pg, pg_name, max_scf_name_size);
2516         if (len < 0) {
2517                 startd_free(pg_name, max_scf_name_size);
2518                 switch (scf_error()) {
2519                 case SCF_ERROR_CONNECTION_BROKEN:
2520                 default:
2521                         return (info->err = ECONNABORTED);
2522 
2523                 case SCF_ERROR_DELETED:
2524                         return (info->err = 0);
2525 
2526                 case SCF_ERROR_NOT_SET:
2527                         bad_error("scf_pg_get_name", scf_error());
2528                 }
2529         }
2530 
2531         /*
2532          * Skip over empty dependency groups.  Since dependency property
2533          * groups are updated atomically, they are either empty or
2534          * fully populated.
2535          */
2536         empty = depgroup_empty(h, pg);
2537         if (empty < 0) {
2538                 log_error(LOG_INFO,
2539                     "Error reading dependency group \"%s\" of %s: %s\n",
2540                     pg_name, info->v->gv_name, scf_strerror(scf_error()));
2541                 startd_free(pg_name, max_scf_name_size);
2542                 return (info->err = EINVAL);
2543 
2544         } else if (empty == 1) {
2545                 log_framework(LOG_DEBUG,
2546                     "Ignoring empty dependency group \"%s\" of %s\n",
2547                     pg_name, info->v->gv_name);
2548                 startd_free(pg_name, max_scf_name_size);
2549                 return (info->err = 0);
2550         }
2551 
2552         fmri_sz = strlen(info->v->gv_name) + 1 + len + 1;
2553         fmri = startd_alloc(fmri_sz);
2554 
2555         (void) snprintf(fmri, fmri_sz, "%s>%s", info->v->gv_name,
2556             pg_name);
2557 
2558         /* Validate the pg before modifying the graph */
2559         deptype = depgroup_read_grouping(h, pg);
2560         if (deptype == DEPGRP_UNSUPPORTED) {
2561                 log_error(LOG_INFO,
2562                     "Dependency \"%s\" of %s has an unknown grouping value.\n",
2563                     pg_name, info->v->gv_name);
2564                 startd_free(fmri, fmri_sz);
2565                 startd_free(pg_name, max_scf_name_size);
2566                 return (info->err = EINVAL);
2567         }
2568 
2569         rerr = depgroup_read_restart(h, pg);
2570         if (rerr == RERR_UNSUPPORTED) {
2571                 log_error(LOG_INFO,
2572                     "Dependency \"%s\" of %s has an unknown restart_on value."
2573                     "\n", pg_name, info->v->gv_name);
2574                 startd_free(fmri, fmri_sz);
2575                 startd_free(pg_name, max_scf_name_size);
2576                 return (info->err = EINVAL);
2577         }
2578 
2579         prop = safe_scf_property_create(h);
2580 
2581         if (scf_pg_get_property(pg, SCF_PROPERTY_ENTITIES, prop) != 0) {
2582                 scferr = scf_error();
2583                 scf_property_destroy(prop);
2584                 if (scferr == SCF_ERROR_DELETED) {
2585                         startd_free(fmri, fmri_sz);
2586                         startd_free(pg_name, max_scf_name_size);
2587                         return (info->err = 0);
2588                 } else if (scferr != SCF_ERROR_NOT_FOUND) {
2589                         startd_free(fmri, fmri_sz);
2590                         startd_free(pg_name, max_scf_name_size);
2591                         return (info->err = ECONNABORTED);
2592                 }
2593 
2594                 log_error(LOG_INFO,
2595                     "Dependency \"%s\" of %s is missing a \"%s\" property.\n",
2596                     pg_name, info->v->gv_name, SCF_PROPERTY_ENTITIES);
2597 
2598                 startd_free(fmri, fmri_sz);
2599                 startd_free(pg_name, max_scf_name_size);
2600 
2601                 return (info->err = EINVAL);
2602         }
2603 
2604         /* Create depgroup vertex for pg */
2605         err = graph_insert_vertex_unconfigured(fmri, GVT_GROUP, deptype,
2606             rerr, &depgrp);
2607         assert(err == 0);
2608         startd_free(fmri, fmri_sz);
2609 
2610         /* Add dependency from inst vertex to new vertex */
2611         err = graph_insert_dependency(info->v, depgrp, info->pathp);
2612         /* ELOOP can't happen because this should be a new vertex */
2613         assert(err == 0);
2614 
2615         linfo.v = depgrp;
2616         linfo.type = depgroup_read_scheme(h, pg);
2617         linfo.inst_fmri = info->v->gv_name;
2618         linfo.pg_name = pg_name;
2619         linfo.h = h;
2620         linfo.err = 0;
2621         linfo.pathp = info->pathp;
2622         err = walk_property_astrings(prop, (callback_t)process_dependency_fmri,
2623             &linfo);
2624 
2625         scf_property_destroy(prop);
2626         startd_free(pg_name, max_scf_name_size);
2627 
2628         switch (err) {
2629         case 0:
2630         case EINTR:
2631                 return (info->err = linfo.err);
2632 
2633         case ECONNABORTED:
2634         case EINVAL:
2635                 return (info->err = err);
2636 
2637         case ECANCELED:
2638                 return (info->err = 0);
2639 
2640         case ECONNRESET:
2641                 return (info->err = ECONNABORTED);
2642 
2643         default:
2644                 bad_error("walk_property_astrings", err);
2645                 /* NOTREACHED */
2646         }
2647 }
2648 
2649 /*
2650  * Build the dependency info for v from the repository.  Returns 0 on success,
2651  * ECONNABORTED on repository disconnection, EINVAL if the repository
2652  * configuration is invalid, and ELOOP if a dependency would cause a cycle.
2653  * In the last case, *pathp will point to a -1-terminated array of ids which
2654  * constitute the rest of the dependency cycle.
2655  */
2656 static int
2657 set_dependencies(graph_vertex_t *v, scf_instance_t *inst, int **pathp)
2658 {
2659         struct deppg_info info;
2660         int err;
2661         uint_t old_configured;
2662 
2663         assert(MUTEX_HELD(&dgraph_lock));
2664 
2665         /*
2666          * Mark the vertex as configured during dependency insertion to avoid
2667          * dependency cycles (which can appear in the graph if one of the
2668          * vertices is an exclusion-group).
2669          */
2670         old_configured = v->gv_flags & GV_CONFIGURED;
2671         v->gv_flags |= GV_CONFIGURED;
2672 
2673         info.err = 0;
2674         info.v = v;
2675         info.pathp = pathp;
2676 
2677         err = walk_dependency_pgs(inst, (callback_t)process_dependency_pg,
2678             &info);
2679 
2680         if (!old_configured)
2681                 v->gv_flags &= ~GV_CONFIGURED;
2682 
2683         switch (err) {
2684         case 0:
2685         case EINTR:
2686                 return (info.err);
2687 
2688         case ECONNABORTED:
2689                 return (ECONNABORTED);
2690 
2691         case ECANCELED:
2692                 /* Should get delete event, so return 0. */
2693                 return (0);
2694 
2695         default:
2696                 bad_error("walk_dependency_pgs", err);
2697                 /* NOTREACHED */
2698         }
2699 }
2700 
2701 
2702 static void
2703 handle_cycle(const char *fmri, int *path)
2704 {
2705         const char *cp;
2706         size_t sz;
2707 
2708         assert(MUTEX_HELD(&dgraph_lock));
2709 
2710         path_to_str(path, (char **)&cp, &sz);
2711 
2712         log_error(LOG_ERR, "Transitioning %s to maintenance "
2713             "because it completes a dependency cycle (see svcs -xv for "
2714             "details):\n%s", fmri ? fmri : "?", cp);
2715 
2716         startd_free((void *)cp, sz);
2717 }
2718 
2719 /*
2720  * Increment the vertex's reference count to prevent the vertex removal
2721  * from the dgraph.
2722  */
2723 static void
2724 vertex_ref(graph_vertex_t *v)
2725 {
2726         assert(MUTEX_HELD(&dgraph_lock));
2727 
2728         v->gv_refs++;
2729 }
2730 
2731 /*
2732  * Decrement the vertex's reference count and remove the vertex from
2733  * the dgraph when possible.
2734  *
2735  * Return VERTEX_REMOVED when the vertex has been removed otherwise
2736  * return VERTEX_INUSE.
2737  */
2738 static int
2739 vertex_unref(graph_vertex_t *v)
2740 {
2741         assert(MUTEX_HELD(&dgraph_lock));
2742         assert(v->gv_refs > 0);
2743 
2744         v->gv_refs--;
2745 
2746         return (free_if_unrefed(v));
2747 }
2748 
2749 /*
2750  * When run on the dependencies of a vertex, populates list with
2751  * graph_edge_t's which point to the service vertices or the instance
2752  * vertices (no GVT_GROUP nodes) on which the vertex depends.
2753  *
2754  * Increment the vertex's reference count once the vertex is inserted
2755  * in the list. The vertex won't be able to be deleted from the dgraph
2756  * while it is referenced.
2757  */
2758 static int
2759 append_svcs_or_insts(graph_edge_t *e, uu_list_t *list)
2760 {
2761         graph_vertex_t *v = e->ge_vertex;
2762         graph_edge_t *new;
2763         int r;
2764 
2765         switch (v->gv_type) {
2766         case GVT_INST:
2767         case GVT_SVC:
2768                 break;
2769 
2770         case GVT_GROUP:
2771                 r = uu_list_walk(v->gv_dependencies,
2772                     (uu_walk_fn_t *)append_svcs_or_insts, list, 0);
2773                 assert(r == 0);
2774                 return (UU_WALK_NEXT);
2775 
2776         case GVT_FILE:
2777                 return (UU_WALK_NEXT);
2778 
2779         default:
2780 #ifndef NDEBUG
2781                 uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
2782                     __LINE__, v->gv_type);
2783 #endif
2784                 abort();
2785         }
2786 
2787         new = startd_alloc(sizeof (*new));
2788         new->ge_vertex = v;
2789         uu_list_node_init(new, &new->ge_link, graph_edge_pool);
2790         r = uu_list_insert_before(list, NULL, new);
2791         assert(r == 0);
2792 
2793         /*
2794          * Because we are inserting the vertex in a list, we don't want
2795          * the vertex to be freed while the list is in use. In order to
2796          * achieve that, increment the vertex's reference count.
2797          */
2798         vertex_ref(v);
2799 
2800         return (UU_WALK_NEXT);
2801 }
2802 
2803 static boolean_t
2804 should_be_in_subgraph(graph_vertex_t *v)
2805 {
2806         graph_edge_t *e;
2807 
2808         if (v == milestone)
2809                 return (B_TRUE);
2810 
2811         /*
2812          * v is in the subgraph if any of its dependents are in the subgraph.
2813          * Except for EXCLUDE_ALL dependents.  And OPTIONAL dependents only
2814          * count if we're enabled.
2815          */
2816         for (e = uu_list_first(v->gv_dependents);
2817             e != NULL;
2818             e = uu_list_next(v->gv_dependents, e)) {
2819                 graph_vertex_t *dv = e->ge_vertex;
2820 
2821                 if (!(dv->gv_flags & GV_INSUBGRAPH))
2822                         continue;
2823 
2824                 /*
2825                  * Don't include instances that are optional and disabled.
2826                  */
2827                 if (v->gv_type == GVT_INST && dv->gv_type == GVT_SVC) {
2828 
2829                         int in = 0;
2830                         graph_edge_t *ee;
2831 
2832                         for (ee = uu_list_first(dv->gv_dependents);
2833                             ee != NULL;
2834                             ee = uu_list_next(dv->gv_dependents, ee)) {
2835 
2836                                 graph_vertex_t *ddv = e->ge_vertex;
2837 
2838                                 if (ddv->gv_type == GVT_GROUP &&
2839                                     ddv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2840                                         continue;
2841 
2842                                 if (ddv->gv_type == GVT_GROUP &&
2843                                     ddv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2844                                     !(v->gv_flags & GV_ENBLD_NOOVR))
2845                                         continue;
2846 
2847                                 in = 1;
2848                         }
2849                         if (!in)
2850                                 continue;
2851                 }
2852                 if (v->gv_type == GVT_INST &&
2853                     dv->gv_type == GVT_GROUP &&
2854                     dv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2855                     !(v->gv_flags & GV_ENBLD_NOOVR))
2856                         continue;
2857 
2858                 /* Don't include excluded services and instances */
2859                 if (dv->gv_type == GVT_GROUP &&
2860                     dv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2861                         continue;
2862 
2863                 return (B_TRUE);
2864         }
2865 
2866         return (B_FALSE);
2867 }
2868 
2869 /*
2870  * Ensures that GV_INSUBGRAPH is set properly for v and its descendents.  If
2871  * any bits change, manipulate the repository appropriately.  Returns 0 or
2872  * ECONNABORTED.
2873  */
2874 static int
2875 eval_subgraph(graph_vertex_t *v, scf_handle_t *h)
2876 {
2877         boolean_t old = (v->gv_flags & GV_INSUBGRAPH) != 0;
2878         boolean_t new;
2879         graph_edge_t *e;
2880         scf_instance_t *inst;
2881         int ret = 0, r;
2882 
2883         assert(milestone != NULL && milestone != MILESTONE_NONE);
2884 
2885         new = should_be_in_subgraph(v);
2886 
2887         if (new == old)
2888                 return (0);
2889 
2890         log_framework(LOG_DEBUG, new ? "Adding %s to the subgraph.\n" :
2891             "Removing %s from the subgraph.\n", v->gv_name);
2892 
2893         v->gv_flags = (v->gv_flags & ~GV_INSUBGRAPH) |
2894             (new ? GV_INSUBGRAPH : 0);
2895 
2896         if (v->gv_type == GVT_INST && (v->gv_flags & GV_CONFIGURED)) {
2897                 int err;
2898 
2899 get_inst:
2900                 err = libscf_fmri_get_instance(h, v->gv_name, &inst);
2901                 if (err != 0) {
2902                         switch (err) {
2903                         case ECONNABORTED:
2904                                 libscf_handle_rebind(h);
2905                                 ret = ECONNABORTED;
2906                                 goto get_inst;
2907 
2908                         case ENOENT:
2909                                 break;
2910 
2911                         case EINVAL:
2912                         case ENOTSUP:
2913                         default:
2914                                 bad_error("libscf_fmri_get_instance", err);
2915                         }
2916                 } else {
2917                         const char *f;
2918 
2919                         if (new) {
2920                                 err = libscf_delete_enable_ovr(inst);
2921                                 f = "libscf_delete_enable_ovr";
2922                         } else {
2923                                 err = libscf_set_enable_ovr(inst, 0);
2924                                 f = "libscf_set_enable_ovr";
2925                         }
2926                         scf_instance_destroy(inst);
2927                         switch (err) {
2928                         case 0:
2929                         case ECANCELED:
2930                                 break;
2931 
2932                         case ECONNABORTED:
2933                                 libscf_handle_rebind(h);
2934                                 /*
2935                                  * We must continue so the graph is updated,
2936                                  * but we must return ECONNABORTED so any
2937                                  * libscf state held by any callers is reset.
2938                                  */
2939                                 ret = ECONNABORTED;
2940                                 goto get_inst;
2941 
2942                         case EROFS:
2943                         case EPERM:
2944                                 log_error(LOG_WARNING,
2945                                     "Could not set %s/%s for %s: %s.\n",
2946                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
2947                                     v->gv_name, strerror(err));
2948                                 break;
2949 
2950                         default:
2951                                 bad_error(f, err);
2952                         }
2953                 }
2954         }
2955 
2956         for (e = uu_list_first(v->gv_dependencies);
2957             e != NULL;
2958             e = uu_list_next(v->gv_dependencies, e)) {
2959                 r = eval_subgraph(e->ge_vertex, h);
2960                 if (r != 0) {
2961                         assert(r == ECONNABORTED);
2962                         ret = ECONNABORTED;
2963                 }
2964         }
2965 
2966         return (ret);
2967 }
2968 
2969 /*
2970  * Delete the (property group) dependencies of v & create new ones based on
2971  * inst.  If doing so would create a cycle, log a message and put the instance
2972  * into maintenance.  Update GV_INSUBGRAPH flags as necessary.  Returns 0 or
2973  * ECONNABORTED.
2974  */
2975 int
2976 refresh_vertex(graph_vertex_t *v, scf_instance_t *inst)
2977 {
2978         int err;
2979         int *path;
2980         char *fmri;
2981         int r;
2982         scf_handle_t *h = scf_instance_handle(inst);
2983         uu_list_t *old_deps;
2984         int ret = 0;
2985         graph_edge_t *e;
2986         graph_vertex_t *vv;
2987 
2988         assert(MUTEX_HELD(&dgraph_lock));
2989         assert(v->gv_type == GVT_INST);
2990 
2991         log_framework(LOG_DEBUG, "Graph engine: Refreshing %s.\n", v->gv_name);
2992 
2993         if (milestone > MILESTONE_NONE) {
2994                 /*
2995                  * In case some of v's dependencies are being deleted we must
2996                  * make a list of them now for GV_INSUBGRAPH-flag evaluation
2997                  * after the new dependencies are in place.
2998                  */
2999                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
3000 
3001                 err = uu_list_walk(v->gv_dependencies,
3002                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
3003                 assert(err == 0);
3004         }
3005 
3006         delete_instance_dependencies(v, B_FALSE);
3007 
3008         err = set_dependencies(v, inst, &path);
3009         switch (err) {
3010         case 0:
3011                 break;
3012 
3013         case ECONNABORTED:
3014                 ret = err;
3015                 goto out;
3016 
3017         case EINVAL:
3018         case ELOOP:
3019                 r = libscf_instance_get_fmri(inst, &fmri);
3020                 switch (r) {
3021                 case 0:
3022                         break;
3023 
3024                 case ECONNABORTED:
3025                         ret = ECONNABORTED;
3026                         goto out;
3027 
3028                 case ECANCELED:
3029                         ret = 0;
3030                         goto out;
3031 
3032                 default:
3033                         bad_error("libscf_instance_get_fmri", r);
3034                 }
3035 
3036                 if (err == EINVAL) {
3037                         log_error(LOG_ERR, "Transitioning %s "
3038                             "to maintenance due to misconfiguration.\n",
3039                             fmri ? fmri : "?");
3040                         vertex_send_event(v,
3041                             RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY);
3042                 } else {
3043                         handle_cycle(fmri, path);
3044                         vertex_send_event(v,
3045                             RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE);
3046                 }
3047                 startd_free(fmri, max_scf_fmri_size);
3048                 ret = 0;
3049                 goto out;
3050 
3051         default:
3052                 bad_error("set_dependencies", err);
3053         }
3054 
3055         if (milestone > MILESTONE_NONE) {
3056                 boolean_t aborted = B_FALSE;
3057 
3058                 for (e = uu_list_first(old_deps);
3059                     e != NULL;
3060                     e = uu_list_next(old_deps, e)) {
3061                         vv = e->ge_vertex;
3062 
3063                         if (vertex_unref(vv) == VERTEX_INUSE &&
3064                             eval_subgraph(vv, h) == ECONNABORTED)
3065                                 aborted = B_TRUE;
3066                 }
3067 
3068                 for (e = uu_list_first(v->gv_dependencies);
3069                     e != NULL;
3070                     e = uu_list_next(v->gv_dependencies, e)) {
3071                         if (eval_subgraph(e->ge_vertex, h) ==
3072                             ECONNABORTED)
3073                                 aborted = B_TRUE;
3074                 }
3075 
3076                 if (aborted) {
3077                         ret = ECONNABORTED;
3078                         goto out;
3079                 }
3080         }
3081 
3082         graph_start_if_satisfied(v);
3083 
3084         ret = 0;
3085 
3086 out:
3087         if (milestone > MILESTONE_NONE) {
3088                 void *cookie = NULL;
3089 
3090                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL)
3091                         startd_free(e, sizeof (*e));
3092 
3093                 uu_list_destroy(old_deps);
3094         }
3095 
3096         return (ret);
3097 }
3098 
3099 /*
3100  * Set up v according to inst.  That is, make sure it depends on its
3101  * restarter and set up its dependencies.  Send the ADD_INSTANCE command to
3102  * the restarter, and send ENABLE or DISABLE as appropriate.
3103  *
3104  * Returns 0 on success, ECONNABORTED on repository disconnection, or
3105  * ECANCELED if inst is deleted.
3106  */
3107 static int
3108 configure_vertex(graph_vertex_t *v, scf_instance_t *inst)
3109 {
3110         scf_handle_t *h;
3111         scf_propertygroup_t *pg;
3112         scf_snapshot_t *snap;
3113         char *restarter_fmri = startd_alloc(max_scf_value_size);
3114         int enabled, enabled_ovr;
3115         int err;
3116         int *path;
3117         int deathrow;
3118         int32_t tset;
3119 
3120         restarter_fmri[0] = '\0';
3121 
3122         assert(MUTEX_HELD(&dgraph_lock));
3123         assert(v->gv_type == GVT_INST);
3124         assert((v->gv_flags & GV_CONFIGURED) == 0);
3125 
3126         /* GV_INSUBGRAPH should already be set properly. */
3127         assert(should_be_in_subgraph(v) ==
3128             ((v->gv_flags & GV_INSUBGRAPH) != 0));
3129 
3130         /*
3131          * If the instance fmri is in the deathrow list then set the
3132          * GV_DEATHROW flag on the vertex and create and set to true the
3133          * SCF_PROPERTY_DEATHROW boolean property in the non-persistent
3134          * repository for this instance fmri.
3135          */
3136         if ((v->gv_flags & GV_DEATHROW) ||
3137             (is_fmri_in_deathrow(v->gv_name) == B_TRUE)) {
3138                 if ((v->gv_flags & GV_DEATHROW) == 0) {
3139                         /*
3140                          * Set flag GV_DEATHROW, create and set to true
3141                          * the SCF_PROPERTY_DEATHROW property in the
3142                          * non-persistent repository for this instance fmri.
3143                          */
3144                         v->gv_flags |= GV_DEATHROW;
3145 
3146                         switch (err = libscf_set_deathrow(inst, 1)) {
3147                         case 0:
3148                                 break;
3149 
3150                         case ECONNABORTED:
3151                         case ECANCELED:
3152                                 startd_free(restarter_fmri, max_scf_value_size);
3153                                 return (err);
3154 
3155                         case EROFS:
3156                                 log_error(LOG_WARNING, "Could not set %s/%s "
3157                                     "for deathrow %s: %s.\n",
3158                                     SCF_PG_DEATHROW, SCF_PROPERTY_DEATHROW,
3159                                     v->gv_name, strerror(err));
3160                                 break;
3161 
3162                         case EPERM:
3163                                 uu_die("Permission denied.\n");
3164                                 /* NOTREACHED */
3165 
3166                         default:
3167                                 bad_error("libscf_set_deathrow", err);
3168                         }
3169                         log_framework(LOG_DEBUG, "Deathrow, graph set %s.\n",
3170                             v->gv_name);
3171                 }
3172                 startd_free(restarter_fmri, max_scf_value_size);
3173                 return (0);
3174         }
3175 
3176         h = scf_instance_handle(inst);
3177 
3178         /*
3179          * Using a temporary deathrow boolean property, set through
3180          * libscf_set_deathrow(), only for fmris on deathrow, is necessary
3181          * because deathrow_fini() may already have been called, and in case
3182          * of a refresh, GV_DEATHROW may need to be set again.
3183          * libscf_get_deathrow() sets deathrow to 1 only if this instance
3184          * has a temporary boolean property named 'deathrow' valued true
3185          * in a property group 'deathrow', -1 or 0 in all other cases.
3186          */
3187         err = libscf_get_deathrow(h, inst, &deathrow);
3188         switch (err) {
3189         case 0:
3190                 break;
3191 
3192         case ECONNABORTED:
3193         case ECANCELED:
3194                 startd_free(restarter_fmri, max_scf_value_size);
3195                 return (err);
3196 
3197         default:
3198                 bad_error("libscf_get_deathrow", err);
3199         }
3200 
3201         if (deathrow == 1) {
3202                 v->gv_flags |= GV_DEATHROW;
3203                 startd_free(restarter_fmri, max_scf_value_size);
3204                 return (0);
3205         }
3206 
3207         log_framework(LOG_DEBUG, "Graph adding %s.\n", v->gv_name);
3208 
3209         /*
3210          * If the instance does not have a restarter property group,
3211          * initialize its state to uninitialized/none, in case the restarter
3212          * is not enabled.
3213          */
3214         pg = safe_scf_pg_create(h);
3215 
3216         if (scf_instance_get_pg(inst, SCF_PG_RESTARTER, pg) != 0) {
3217                 instance_data_t idata;
3218                 uint_t count = 0, msecs = ALLOC_DELAY;
3219 
3220                 switch (scf_error()) {
3221                 case SCF_ERROR_NOT_FOUND:
3222                         break;
3223 
3224                 case SCF_ERROR_CONNECTION_BROKEN:
3225                 default:
3226                         scf_pg_destroy(pg);
3227                         startd_free(restarter_fmri, max_scf_value_size);
3228                         return (ECONNABORTED);
3229 
3230                 case SCF_ERROR_DELETED:
3231                         scf_pg_destroy(pg);
3232                         startd_free(restarter_fmri, max_scf_value_size);
3233                         return (ECANCELED);
3234 
3235                 case SCF_ERROR_NOT_SET:
3236                         bad_error("scf_instance_get_pg", scf_error());
3237                 }
3238 
3239                 switch (err = libscf_instance_get_fmri(inst,
3240                     (char **)&idata.i_fmri)) {
3241                 case 0:
3242                         break;
3243 
3244                 case ECONNABORTED:
3245                 case ECANCELED:
3246                         scf_pg_destroy(pg);
3247                         startd_free(restarter_fmri, max_scf_value_size);
3248                         return (err);
3249 
3250                 default:
3251                         bad_error("libscf_instance_get_fmri", err);
3252                 }
3253 
3254                 idata.i_state = RESTARTER_STATE_NONE;
3255                 idata.i_next_state = RESTARTER_STATE_NONE;
3256 
3257 init_state:
3258                 switch (err = _restarter_commit_states(h, &idata,
3259                     RESTARTER_STATE_UNINIT, RESTARTER_STATE_NONE,
3260                     restarter_get_str_short(restarter_str_insert_in_graph))) {
3261                 case 0:
3262                         break;
3263 
3264                 case ENOMEM:
3265                         ++count;
3266                         if (count < ALLOC_RETRY) {
3267                                 (void) poll(NULL, 0, msecs);
3268                                 msecs *= ALLOC_DELAY_MULT;
3269                                 goto init_state;
3270                         }
3271 
3272                         uu_die("Insufficient memory.\n");
3273                         /* NOTREACHED */
3274 
3275                 case ECONNABORTED:
3276                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3277                         scf_pg_destroy(pg);
3278                         startd_free(restarter_fmri, max_scf_value_size);
3279                         return (ECONNABORTED);
3280 
3281                 case ENOENT:
3282                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3283                         scf_pg_destroy(pg);
3284                         startd_free(restarter_fmri, max_scf_value_size);
3285                         return (ECANCELED);
3286 
3287                 case EPERM:
3288                 case EACCES:
3289                 case EROFS:
3290                         log_error(LOG_NOTICE, "Could not initialize state for "
3291                             "%s: %s.\n", idata.i_fmri, strerror(err));
3292                         break;
3293 
3294                 case EINVAL:
3295                 default:
3296                         bad_error("_restarter_commit_states", err);
3297                 }
3298 
3299                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3300         }
3301 
3302         scf_pg_destroy(pg);
3303 
3304         if (milestone != NULL) {
3305                 /*
3306                  * Make sure the enable-override is set properly before we
3307                  * read whether we should be enabled.
3308                  */
3309                 if (milestone == MILESTONE_NONE ||
3310                     !(v->gv_flags & GV_INSUBGRAPH)) {
3311                         /*
3312                          * This might seem unjustified after the milestone
3313                          * transition has completed (non_subgraph_svcs == 0),
3314                          * but it's important because when we boot to
3315                          * a milestone, we set the milestone before populating
3316                          * the graph, and all of the new non-subgraph services
3317                          * need to be disabled here.
3318                          */
3319                         switch (err = libscf_set_enable_ovr(inst, 0)) {
3320                         case 0:
3321                                 break;
3322 
3323                         case ECONNABORTED:
3324                         case ECANCELED:
3325                                 startd_free(restarter_fmri, max_scf_value_size);
3326                                 return (err);
3327 
3328                         case EROFS:
3329                                 log_error(LOG_WARNING,
3330                                     "Could not set %s/%s for %s: %s.\n",
3331                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
3332                                     v->gv_name, strerror(err));
3333                                 break;
3334 
3335                         case EPERM:
3336                                 uu_die("Permission denied.\n");
3337                                 /* NOTREACHED */
3338 
3339                         default:
3340                                 bad_error("libscf_set_enable_ovr", err);
3341                         }
3342                 } else {
3343                         assert(v->gv_flags & GV_INSUBGRAPH);
3344                         switch (err = libscf_delete_enable_ovr(inst)) {
3345                         case 0:
3346                                 break;
3347 
3348                         case ECONNABORTED:
3349                         case ECANCELED:
3350                                 startd_free(restarter_fmri, max_scf_value_size);
3351                                 return (err);
3352 
3353                         case EPERM:
3354                                 uu_die("Permission denied.\n");
3355                                 /* NOTREACHED */
3356 
3357                         default:
3358                                 bad_error("libscf_delete_enable_ovr", err);
3359                         }
3360                 }
3361         }
3362 
3363         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
3364             &enabled_ovr, &restarter_fmri);
3365         switch (err) {
3366         case 0:
3367                 break;
3368 
3369         case ECONNABORTED:
3370         case ECANCELED:
3371                 startd_free(restarter_fmri, max_scf_value_size);
3372                 return (err);
3373 
3374         case ENOENT:
3375                 log_framework(LOG_DEBUG,
3376                     "Ignoring %s because it has no general property group.\n",
3377                     v->gv_name);
3378                 startd_free(restarter_fmri, max_scf_value_size);
3379                 return (0);
3380 
3381         default:
3382                 bad_error("libscf_get_basic_instance_data", err);
3383         }
3384 
3385         if ((tset = libscf_get_stn_tset(inst)) == -1) {
3386                 log_framework(LOG_WARNING,
3387                     "Failed to get notification parameters for %s: %s\n",
3388                     v->gv_name, scf_strerror(scf_error()));
3389                 v->gv_stn_tset = 0;
3390         } else {
3391                 v->gv_stn_tset = tset;
3392         }
3393         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
3394                 stn_global = v->gv_stn_tset;
3395 
3396         if (enabled == -1) {
3397                 startd_free(restarter_fmri, max_scf_value_size);
3398                 return (0);
3399         }
3400 
3401         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
3402             (enabled ? GV_ENBLD_NOOVR : 0);
3403 
3404         if (enabled_ovr != -1)
3405                 enabled = enabled_ovr;
3406 
3407         v->gv_state = RESTARTER_STATE_UNINIT;
3408 
3409         snap = libscf_get_or_make_running_snapshot(inst, v->gv_name, B_TRUE);
3410         scf_snapshot_destroy(snap);
3411 
3412         /* Set up the restarter. (Sends _ADD_INSTANCE on success.) */
3413         err = graph_change_restarter(v, restarter_fmri, h, &path);
3414         if (err != 0) {
3415                 instance_data_t idata;
3416                 uint_t count = 0, msecs = ALLOC_DELAY;
3417                 restarter_str_t reason;
3418 
3419                 if (err == ECONNABORTED) {
3420                         startd_free(restarter_fmri, max_scf_value_size);
3421                         return (err);
3422                 }
3423 
3424                 assert(err == EINVAL || err == ELOOP);
3425 
3426                 if (err == EINVAL) {
3427                         log_framework(LOG_ERR, emsg_invalid_restarter,
3428                             v->gv_name, restarter_fmri);
3429                         reason = restarter_str_invalid_restarter;
3430                 } else {
3431                         handle_cycle(v->gv_name, path);
3432                         reason = restarter_str_dependency_cycle;
3433                 }
3434 
3435                 startd_free(restarter_fmri, max_scf_value_size);
3436 
3437                 /*
3438                  * We didn't register the instance with the restarter, so we
3439                  * must set maintenance mode ourselves.
3440                  */
3441                 err = libscf_instance_get_fmri(inst, (char **)&idata.i_fmri);
3442                 if (err != 0) {
3443                         assert(err == ECONNABORTED || err == ECANCELED);
3444                         return (err);
3445                 }
3446 
3447                 idata.i_state = RESTARTER_STATE_NONE;
3448                 idata.i_next_state = RESTARTER_STATE_NONE;
3449 
3450 set_maint:
3451                 switch (err = _restarter_commit_states(h, &idata,
3452                     RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE,
3453                     restarter_get_str_short(reason))) {
3454                 case 0:
3455                         break;
3456 
3457                 case ENOMEM:
3458                         ++count;
3459                         if (count < ALLOC_RETRY) {
3460                                 (void) poll(NULL, 0, msecs);
3461                                 msecs *= ALLOC_DELAY_MULT;
3462                                 goto set_maint;
3463                         }
3464 
3465                         uu_die("Insufficient memory.\n");
3466                         /* NOTREACHED */
3467 
3468                 case ECONNABORTED:
3469                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3470                         return (ECONNABORTED);
3471 
3472                 case ENOENT:
3473                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3474                         return (ECANCELED);
3475 
3476                 case EPERM:
3477                 case EACCES:
3478                 case EROFS:
3479                         log_error(LOG_NOTICE, "Could not initialize state for "
3480                             "%s: %s.\n", idata.i_fmri, strerror(err));
3481                         break;
3482 
3483                 case EINVAL:
3484                 default:
3485                         bad_error("_restarter_commit_states", err);
3486                 }
3487 
3488                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3489 
3490                 v->gv_state = RESTARTER_STATE_MAINT;
3491 
3492                 goto out;
3493         }
3494         startd_free(restarter_fmri, max_scf_value_size);
3495 
3496         /* Add all the other dependencies. */
3497         err = refresh_vertex(v, inst);
3498         if (err != 0) {
3499                 assert(err == ECONNABORTED);
3500                 return (err);
3501         }
3502 
3503 out:
3504         v->gv_flags |= GV_CONFIGURED;
3505 
3506         graph_enable_by_vertex(v, enabled, 0);
3507 
3508         return (0);
3509 }
3510 
3511 
3512 static void
3513 kill_user_procs(void)
3514 {
3515         (void) fputs("svc.startd: Killing user processes.\n", stdout);
3516 
3517         /*
3518          * Despite its name, killall's role is to get select user processes--
3519          * basically those representing terminal-based logins-- to die.  Victims
3520          * are located by killall in the utmp database.  Since these are most
3521          * often shell based logins, and many shells mask SIGTERM (but are
3522          * responsive to SIGHUP) we first HUP and then shortly thereafter
3523          * kill -9.
3524          */
3525         (void) fork_with_timeout("/usr/sbin/killall HUP", 1, 5);
3526         (void) fork_with_timeout("/usr/sbin/killall KILL", 1, 5);
3527 
3528         /*
3529          * Note the selection of user id's 0, 1 and 15, subsequently
3530          * inverted by -v.  15 is reserved for dladmd.  Yes, this is a
3531          * kludge-- a better policy is needed.
3532          *
3533          * Note that fork_with_timeout will only wait out the 1 second
3534          * "grace time" if pkill actually returns 0.  So if there are
3535          * no matches, this will run to completion much more quickly.
3536          */
3537         (void) fork_with_timeout("/usr/bin/pkill -TERM -v -u 0,1,15", 1, 5);
3538         (void) fork_with_timeout("/usr/bin/pkill -KILL -v -u 0,1,15", 1, 5);
3539 }
3540 
3541 static void
3542 do_uadmin(void)
3543 {
3544         const char * const resetting = "/etc/svc/volatile/resetting";
3545         int fd;
3546         struct statvfs vfs;
3547         time_t now;
3548         struct tm nowtm;
3549         char down_buf[256], time_buf[256];
3550         uintptr_t mdep;
3551 #if defined(__i386)
3552         grub_boot_args_t fbarg;
3553 #endif  /* __i386 */
3554 
3555         mdep = NULL;
3556         fd = creat(resetting, 0777);
3557         if (fd >= 0)
3558                 startd_close(fd);
3559         else
3560                 uu_warn("Could not create \"%s\"", resetting);
3561 
3562         /* Kill dhcpagent if we're not using nfs for root */
3563         if ((statvfs("/", &vfs) == 0) &&
3564             (strncmp(vfs.f_basetype, "nfs", sizeof ("nfs") - 1) != 0))
3565                 fork_with_timeout("/usr/bin/pkill -x -u 0 dhcpagent", 0, 5);
3566 
3567         /*
3568          * Call sync(2) now, before we kill off user processes.  This takes
3569          * advantage of the several seconds of pause we have before the
3570          * killalls are done.  Time we can make good use of to get pages
3571          * moving out to disk.
3572          *
3573          * Inside non-global zones, we don't bother, and it's better not to
3574          * anyway, since sync(2) can have system-wide impact.
3575          */
3576         if (getzoneid() == 0)
3577                 sync();
3578 
3579         kill_user_procs();
3580 
3581         /*
3582          * Note that this must come after the killing of user procs, since
3583          * killall relies on utmpx, and this command affects the contents of
3584          * said file.
3585          */
3586         if (access("/usr/lib/acct/closewtmp", X_OK) == 0)
3587                 fork_with_timeout("/usr/lib/acct/closewtmp", 0, 5);
3588 
3589         /*
3590          * For patches which may be installed as the system is shutting
3591          * down, we need to ensure, one more time, that the boot archive
3592          * really is up to date.
3593          */
3594         if (getzoneid() == 0 && access("/usr/sbin/bootadm", X_OK) == 0)
3595                 fork_with_timeout("/usr/sbin/bootadm -ea update_all", 0, 3600);
3596 
3597         /*
3598          * Right now, fast reboot is supported only on i386.
3599          * scf_is_fastboot_default() should take care of it.
3600          * If somehow we got there on unsupported platform -
3601          * print warning and fall back to regular reboot.
3602          */
3603         if (halting == AD_FASTREBOOT) {
3604 #if defined(__i386)
3605                 int rc;
3606 
3607                 if ((rc = grub_get_boot_args(&fbarg, NULL,
3608                     GRUB_ENTRY_DEFAULT)) == 0) {
3609                         mdep = (uintptr_t)&fbarg.gba_bootargs;
3610                 } else {
3611                         /*
3612                          * Failed to read GRUB menu, fall back to normal reboot
3613                          */
3614                         halting = AD_BOOT;
3615                         uu_warn("Failed to process GRUB menu entry "
3616                             "for fast reboot.\n\t%s\n"
3617                             "Falling back to regular reboot.\n",
3618                             grub_strerror(rc));
3619                 }
3620 #else   /* __i386 */
3621                 halting = AD_BOOT;
3622                 uu_warn("Fast reboot configured, but not supported by "
3623                     "this ISA\n");
3624 #endif  /* __i386 */
3625         }
3626 
3627         fork_with_timeout("/sbin/umountall -l", 0, 5);
3628         fork_with_timeout("/sbin/umount /tmp /var/adm /var/run /var "
3629             ">/dev/null 2>&1", 0, 5);
3630 
3631         /*
3632          * Try to get to consistency for whatever UFS filesystems are left.
3633          * This is pretty expensive, so we save it for the end in the hopes of
3634          * minimizing what it must do.  The other option would be to start in
3635          * parallel with the killall's, but lockfs tends to throw out much more
3636          * than is needed, and so subsequent commands (like umountall) take a
3637          * long time to get going again.
3638          *
3639          * Inside of zones, we don't bother, since we're not about to terminate
3640          * the whole OS instance.
3641          *
3642          * On systems using only ZFS, this call to lockfs -fa is a no-op.
3643          */
3644         if (getzoneid() == 0) {
3645                 if (access("/usr/sbin/lockfs", X_OK) == 0)
3646                         fork_with_timeout("/usr/sbin/lockfs -fa", 0, 30);
3647 
3648                 sync(); /* once more, with feeling */
3649         }
3650 
3651         fork_with_timeout("/sbin/umount /usr >/dev/null 2>&1", 0, 5);
3652 
3653         /*
3654          * Construct and emit the last words from userland:
3655          * "<timestamp> The system is down.  Shutdown took <N> seconds."
3656          *
3657          * Normally we'd use syslog, but with /var and other things
3658          * potentially gone, try to minimize the external dependencies.
3659          */
3660         now = time(NULL);
3661         (void) localtime_r(&now, &nowtm);
3662 
3663         if (strftime(down_buf, sizeof (down_buf),
3664             "%b %e %T The system is down.", &nowtm) == 0) {
3665                 (void) strlcpy(down_buf, "The system is down.",
3666                     sizeof (down_buf));
3667         }
3668 
3669         if (halting_time != 0 && halting_time <= now) {
3670                 (void) snprintf(time_buf, sizeof (time_buf),
3671                     "  Shutdown took %lu seconds.", now - halting_time);
3672         } else {
3673                 time_buf[0] = '\0';
3674         }
3675         (void) printf("%s%s\n", down_buf, time_buf);
3676 
3677         (void) uadmin(A_SHUTDOWN, halting, mdep);
3678         uu_warn("uadmin() failed");
3679 
3680 #if defined(__i386)
3681         /* uadmin fail, cleanup grub_boot_args */
3682         if (halting == AD_FASTREBOOT)
3683                 grub_cleanup_boot_args(&fbarg);
3684 #endif  /* __i386 */
3685 
3686         if (remove(resetting) != 0 && errno != ENOENT)
3687                 uu_warn("Could not remove \"%s\"", resetting);
3688 }
3689 
3690 /*
3691  * If any of the up_svcs[] are online or satisfiable, return true.  If they are
3692  * all missing, disabled, in maintenance, or unsatisfiable, return false.
3693  */
3694 boolean_t
3695 can_come_up(void)
3696 {
3697         int i;
3698 
3699         assert(MUTEX_HELD(&dgraph_lock));
3700 
3701         /*
3702          * If we are booting to single user (boot -s),
3703          * SCF_MILESTONE_SINGLE_USER is needed to come up because startd
3704          * spawns sulogin after single-user is online (see specials.c).
3705          */
3706         i = (booting_to_single_user ? 0 : 1);
3707 
3708         for (; up_svcs[i] != NULL; ++i) {
3709                 if (up_svcs_p[i] == NULL) {
3710                         up_svcs_p[i] = vertex_get_by_name(up_svcs[i]);
3711 
3712                         if (up_svcs_p[i] == NULL)
3713                                 continue;
3714                 }
3715 
3716                 /*
3717                  * Ignore unconfigured services (the ones that have been
3718                  * mentioned in a dependency from other services, but do
3719                  * not exist in the repository).  Services which exist
3720                  * in the repository but don't have general/enabled
3721                  * property will be also ignored.
3722                  */
3723                 if (!(up_svcs_p[i]->gv_flags & GV_CONFIGURED))
3724                         continue;
3725 
3726                 switch (up_svcs_p[i]->gv_state) {
3727                 case RESTARTER_STATE_ONLINE:
3728                 case RESTARTER_STATE_DEGRADED:
3729                         /*
3730                          * Deactivate verbose boot once a login service has been
3731                          * reached.
3732                          */
3733                         st->st_log_login_reached = 1;
3734                         /*FALLTHROUGH*/
3735                 case RESTARTER_STATE_UNINIT:
3736                         return (B_TRUE);
3737 
3738                 case RESTARTER_STATE_OFFLINE:
3739                         if (instance_satisfied(up_svcs_p[i], B_TRUE) != -1)
3740                                 return (B_TRUE);
3741                         log_framework(LOG_DEBUG,
3742                             "can_come_up(): %s is unsatisfiable.\n",
3743                             up_svcs_p[i]->gv_name);
3744                         continue;
3745 
3746                 case RESTARTER_STATE_DISABLED:
3747                 case RESTARTER_STATE_MAINT:
3748                         log_framework(LOG_DEBUG,
3749                             "can_come_up(): %s is in state %s.\n",
3750                             up_svcs_p[i]->gv_name,
3751                             instance_state_str[up_svcs_p[i]->gv_state]);
3752                         continue;
3753 
3754                 default:
3755 #ifndef NDEBUG
3756                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
3757                             __FILE__, __LINE__, up_svcs_p[i]->gv_state);
3758 #endif
3759                         abort();
3760                 }
3761         }
3762 
3763         /*
3764          * In the seed repository, console-login is unsatisfiable because
3765          * services are missing.  To behave correctly in that case we don't want
3766          * to return false until manifest-import is online.
3767          */
3768 
3769         if (manifest_import_p == NULL) {
3770                 manifest_import_p = vertex_get_by_name(manifest_import);
3771 
3772                 if (manifest_import_p == NULL)
3773                         return (B_FALSE);
3774         }
3775 
3776         switch (manifest_import_p->gv_state) {
3777         case RESTARTER_STATE_ONLINE:
3778         case RESTARTER_STATE_DEGRADED:
3779         case RESTARTER_STATE_DISABLED:
3780         case RESTARTER_STATE_MAINT:
3781                 break;
3782 
3783         case RESTARTER_STATE_OFFLINE:
3784                 if (instance_satisfied(manifest_import_p, B_TRUE) == -1)
3785                         break;
3786                 /* FALLTHROUGH */
3787 
3788         case RESTARTER_STATE_UNINIT:
3789                 return (B_TRUE);
3790         }
3791 
3792         return (B_FALSE);
3793 }
3794 
3795 /*
3796  * Runs sulogin.  Returns
3797  *   0 - success
3798  *   EALREADY - sulogin is already running
3799  *   EBUSY - console-login is running
3800  */
3801 static int
3802 run_sulogin(const char *msg)
3803 {
3804         graph_vertex_t *v;
3805 
3806         assert(MUTEX_HELD(&dgraph_lock));
3807 
3808         if (sulogin_running)
3809                 return (EALREADY);
3810 
3811         v = vertex_get_by_name(console_login_fmri);
3812         if (v != NULL && inst_running(v))
3813                 return (EBUSY);
3814 
3815         sulogin_running = B_TRUE;
3816 
3817         MUTEX_UNLOCK(&dgraph_lock);
3818 
3819         fork_sulogin(B_FALSE, msg);
3820 
3821         MUTEX_LOCK(&dgraph_lock);
3822 
3823         sulogin_running = B_FALSE;
3824 
3825         if (console_login_ready) {
3826                 v = vertex_get_by_name(console_login_fmri);
3827 
3828                 if (v != NULL && v->gv_state == RESTARTER_STATE_OFFLINE) {
3829                         if (v->gv_start_f == NULL)
3830                                 vertex_send_event(v,
3831                                     RESTARTER_EVENT_TYPE_START);
3832                         else
3833                                 v->gv_start_f(v);
3834                 }
3835 
3836                 console_login_ready = B_FALSE;
3837         }
3838 
3839         return (0);
3840 }
3841 
3842 /*
3843  * The sulogin thread runs sulogin while can_come_up() is false.  run_sulogin()
3844  * keeps sulogin from stepping on console-login's toes.
3845  */
3846 /* ARGSUSED */
3847 static void *
3848 sulogin_thread(void *unused)
3849 {
3850         MUTEX_LOCK(&dgraph_lock);
3851 
3852         assert(sulogin_thread_running);
3853 
3854         do {
3855                 (void) run_sulogin("Console login service(s) cannot run\n");
3856         } while (!can_come_up());
3857 
3858         sulogin_thread_running = B_FALSE;
3859         MUTEX_UNLOCK(&dgraph_lock);
3860 
3861         return (NULL);
3862 }
3863 
3864 /* ARGSUSED */
3865 void *
3866 single_user_thread(void *unused)
3867 {
3868         uint_t left;
3869         scf_handle_t *h;
3870         scf_instance_t *inst;
3871         scf_property_t *prop;
3872         scf_value_t *val;
3873         const char *msg;
3874         char *buf;
3875         int r;
3876 
3877         MUTEX_LOCK(&single_user_thread_lock);
3878         single_user_thread_count++;
3879 
3880         if (!booting_to_single_user)
3881                 kill_user_procs();
3882 
3883         if (go_single_user_mode || booting_to_single_user) {
3884                 msg = "SINGLE USER MODE\n";
3885         } else {
3886                 assert(go_to_level1);
3887 
3888                 fork_rc_script('1', "start", B_TRUE);
3889 
3890                 uu_warn("The system is ready for administration.\n");
3891 
3892                 msg = "";
3893         }
3894 
3895         MUTEX_UNLOCK(&single_user_thread_lock);
3896 
3897         for (;;) {
3898                 MUTEX_LOCK(&dgraph_lock);
3899                 r = run_sulogin(msg);
3900                 MUTEX_UNLOCK(&dgraph_lock);
3901                 if (r == 0)
3902                         break;
3903 
3904                 assert(r == EALREADY || r == EBUSY);
3905 
3906                 left = 3;
3907                 while (left > 0)
3908                         left = sleep(left);
3909         }
3910 
3911         MUTEX_LOCK(&single_user_thread_lock);
3912 
3913         /*
3914          * If another single user thread has started, let it finish changing
3915          * the run level.
3916          */
3917         if (single_user_thread_count > 1) {
3918                 single_user_thread_count--;
3919                 MUTEX_UNLOCK(&single_user_thread_lock);
3920                 return (NULL);
3921         }
3922 
3923         h = libscf_handle_create_bound_loop();
3924         inst = scf_instance_create(h);
3925         prop = safe_scf_property_create(h);
3926         val = safe_scf_value_create(h);
3927         buf = startd_alloc(max_scf_fmri_size);
3928 
3929 lookup:
3930         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
3931             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
3932                 switch (scf_error()) {
3933                 case SCF_ERROR_NOT_FOUND:
3934                         r = libscf_create_self(h);
3935                         if (r == 0)
3936                                 goto lookup;
3937                         assert(r == ECONNABORTED);
3938                         /* FALLTHROUGH */
3939 
3940                 case SCF_ERROR_CONNECTION_BROKEN:
3941                         libscf_handle_rebind(h);
3942                         goto lookup;
3943 
3944                 case SCF_ERROR_INVALID_ARGUMENT:
3945                 case SCF_ERROR_CONSTRAINT_VIOLATED:
3946                 case SCF_ERROR_NOT_BOUND:
3947                 case SCF_ERROR_HANDLE_MISMATCH:
3948                 default:
3949                         bad_error("scf_handle_decode_fmri", scf_error());
3950                 }
3951         }
3952 
3953         MUTEX_LOCK(&dgraph_lock);
3954 
3955         r = scf_instance_delete_prop(inst, SCF_PG_OPTIONS_OVR,
3956             SCF_PROPERTY_MILESTONE);
3957         switch (r) {
3958         case 0:
3959         case ECANCELED:
3960                 break;
3961 
3962         case ECONNABORTED:
3963                 MUTEX_UNLOCK(&dgraph_lock);
3964                 libscf_handle_rebind(h);
3965                 goto lookup;
3966 
3967         case EPERM:
3968         case EACCES:
3969         case EROFS:
3970                 log_error(LOG_WARNING, "Could not clear temporary milestone: "
3971                     "%s.\n", strerror(r));
3972                 break;
3973 
3974         default:
3975                 bad_error("scf_instance_delete_prop", r);
3976         }
3977 
3978         MUTEX_UNLOCK(&dgraph_lock);
3979 
3980         r = libscf_get_milestone(inst, prop, val, buf, max_scf_fmri_size);
3981         switch (r) {
3982         case ECANCELED:
3983         case ENOENT:
3984         case EINVAL:
3985                 (void) strcpy(buf, "all");
3986                 /* FALLTHROUGH */
3987 
3988         case 0:
3989                 uu_warn("Returning to milestone %s.\n", buf);
3990                 break;
3991 
3992         case ECONNABORTED:
3993                 libscf_handle_rebind(h);
3994                 goto lookup;
3995 
3996         default:
3997                 bad_error("libscf_get_milestone", r);
3998         }
3999 
4000         r = dgraph_set_milestone(buf, h, B_FALSE);
4001         switch (r) {
4002         case 0:
4003         case ECONNRESET:
4004         case EALREADY:
4005         case EINVAL:
4006         case ENOENT:
4007                 break;
4008 
4009         default:
4010                 bad_error("dgraph_set_milestone", r);
4011         }
4012 
4013         /*
4014          * See graph_runlevel_changed().
4015          */
4016         MUTEX_LOCK(&dgraph_lock);
4017         utmpx_set_runlevel(target_milestone_as_runlevel(), 'S', B_TRUE);
4018         MUTEX_UNLOCK(&dgraph_lock);
4019 
4020         startd_free(buf, max_scf_fmri_size);
4021         scf_value_destroy(val);
4022         scf_property_destroy(prop);
4023         scf_instance_destroy(inst);
4024         scf_handle_destroy(h);
4025 
4026         /*
4027          * We'll give ourselves 3 seconds to respond to all of the enablings
4028          * that setting the milestone should have created before checking
4029          * whether to run sulogin.
4030          */
4031         left = 3;
4032         while (left > 0)
4033                 left = sleep(left);
4034 
4035         MUTEX_LOCK(&dgraph_lock);
4036         /*
4037          * Clearing these variables will allow the sulogin thread to run.  We
4038          * check here in case there aren't any more state updates anytime soon.
4039          */
4040         go_to_level1 = go_single_user_mode = booting_to_single_user = B_FALSE;
4041         if (!sulogin_thread_running && !can_come_up()) {
4042                 (void) startd_thread_create(sulogin_thread, NULL);
4043                 sulogin_thread_running = B_TRUE;
4044         }
4045         MUTEX_UNLOCK(&dgraph_lock);
4046         single_user_thread_count--;
4047         MUTEX_UNLOCK(&single_user_thread_lock);
4048         return (NULL);
4049 }
4050 
4051 
4052 /*
4053  * Dependency graph operations API.  These are handle-independent thread-safe
4054  * graph manipulation functions which are the entry points for the event
4055  * threads below.
4056  */
4057 
4058 /*
4059  * If a configured vertex exists for inst_fmri, return EEXIST.  If no vertex
4060  * exists for inst_fmri, add one.  Then fetch the restarter from inst, make
4061  * this vertex dependent on it, and send _ADD_INSTANCE to the restarter.
4062  * Fetch whether the instance should be enabled from inst and send _ENABLE or
4063  * _DISABLE as appropriate.  Finally rummage through inst's dependency
4064  * property groups and add vertices and edges as appropriate.  If anything
4065  * goes wrong after sending _ADD_INSTANCE, send _ADMIN_MAINT_ON to put the
4066  * instance in maintenance.  Don't send _START or _STOP until we get a state
4067  * update in case we're being restarted and the service is already running.
4068  *
4069  * To support booting to a milestone, we must also make sure all dependencies
4070  * encountered are configured, if they exist in the repository.
4071  *
4072  * Returns 0 on success, ECONNABORTED on repository disconnection, EINVAL if
4073  * inst_fmri is an invalid (or not canonical) FMRI, ECANCELED if inst is
4074  * deleted, or EEXIST if a configured vertex for inst_fmri already exists.
4075  */
4076 int
4077 dgraph_add_instance(const char *inst_fmri, scf_instance_t *inst,
4078     boolean_t lock_graph)
4079 {
4080         graph_vertex_t *v;
4081         int err;
4082 
4083         if (strcmp(inst_fmri, SCF_SERVICE_STARTD) == 0)
4084                 return (0);
4085 
4086         /* Check for a vertex for inst_fmri. */
4087         if (lock_graph) {
4088                 MUTEX_LOCK(&dgraph_lock);
4089         } else {
4090                 assert(MUTEX_HELD(&dgraph_lock));
4091         }
4092 
4093         v = vertex_get_by_name(inst_fmri);
4094 
4095         if (v != NULL) {
4096                 assert(v->gv_type == GVT_INST);
4097 
4098                 if (v->gv_flags & GV_CONFIGURED) {
4099                         if (lock_graph)
4100                                 MUTEX_UNLOCK(&dgraph_lock);
4101                         return (EEXIST);
4102                 }
4103         } else {
4104                 /* Add the vertex. */
4105                 err = graph_insert_vertex_unconfigured(inst_fmri, GVT_INST, 0,
4106                     RERR_NONE, &v);
4107                 if (err != 0) {
4108                         assert(err == EINVAL);
4109                         if (lock_graph)
4110                                 MUTEX_UNLOCK(&dgraph_lock);
4111                         return (EINVAL);
4112                 }
4113         }
4114 
4115         err = configure_vertex(v, inst);
4116 
4117         if (lock_graph)
4118                 MUTEX_UNLOCK(&dgraph_lock);
4119 
4120         return (err);
4121 }
4122 
4123 /*
4124  * Locate the vertex for this property group's instance.  If it doesn't exist
4125  * or is unconfigured, call dgraph_add_instance() & return.  Otherwise fetch
4126  * the restarter for the instance, and if it has changed, send
4127  * _REMOVE_INSTANCE to the old restarter, remove the dependency, make sure the
4128  * new restarter has a vertex, add a new dependency, and send _ADD_INSTANCE to
4129  * the new restarter.  Then fetch whether the instance should be enabled, and
4130  * if it is different from what we had, or if we changed the restarter, send
4131  * the appropriate _ENABLE or _DISABLE command.
4132  *
4133  * Returns 0 on success, ENOTSUP if the pg's parent is not an instance,
4134  * ECONNABORTED on repository disconnection, ECANCELED if the instance is
4135  * deleted, or -1 if the instance's general property group is deleted or if
4136  * its enabled property is misconfigured.
4137  */
4138 static int
4139 dgraph_update_general(scf_propertygroup_t *pg)
4140 {
4141         scf_handle_t *h;
4142         scf_instance_t *inst;
4143         char *fmri;
4144         char *restarter_fmri;
4145         graph_vertex_t *v;
4146         int err;
4147         int enabled, enabled_ovr;
4148         int oldflags;
4149 
4150         /* Find the vertex for this service */
4151         h = scf_pg_handle(pg);
4152 
4153         inst = safe_scf_instance_create(h);
4154 
4155         if (scf_pg_get_parent_instance(pg, inst) != 0) {
4156                 switch (scf_error()) {
4157                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4158                         return (ENOTSUP);
4159 
4160                 case SCF_ERROR_CONNECTION_BROKEN:
4161                 default:
4162                         return (ECONNABORTED);
4163 
4164                 case SCF_ERROR_DELETED:
4165                         return (0);
4166 
4167                 case SCF_ERROR_NOT_SET:
4168                         bad_error("scf_pg_get_parent_instance", scf_error());
4169                 }
4170         }
4171 
4172         err = libscf_instance_get_fmri(inst, &fmri);
4173         switch (err) {
4174         case 0:
4175                 break;
4176 
4177         case ECONNABORTED:
4178                 scf_instance_destroy(inst);
4179                 return (ECONNABORTED);
4180 
4181         case ECANCELED:
4182                 scf_instance_destroy(inst);
4183                 return (0);
4184 
4185         default:
4186                 bad_error("libscf_instance_get_fmri", err);
4187         }
4188 
4189         log_framework(LOG_DEBUG,
4190             "Graph engine: Reloading general properties for %s.\n", fmri);
4191 
4192         MUTEX_LOCK(&dgraph_lock);
4193 
4194         v = vertex_get_by_name(fmri);
4195         if (v == NULL || !(v->gv_flags & GV_CONFIGURED)) {
4196                 /* Will get the up-to-date properties. */
4197                 MUTEX_UNLOCK(&dgraph_lock);
4198                 err = dgraph_add_instance(fmri, inst, B_TRUE);
4199                 startd_free(fmri, max_scf_fmri_size);
4200                 scf_instance_destroy(inst);
4201                 return (err == ECANCELED ? 0 : err);
4202         }
4203 
4204         /* Read enabled & restarter from repository. */
4205         restarter_fmri = startd_alloc(max_scf_value_size);
4206         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
4207             &enabled_ovr, &restarter_fmri);
4208         if (err != 0 || enabled == -1) {
4209                 MUTEX_UNLOCK(&dgraph_lock);
4210                 scf_instance_destroy(inst);
4211                 startd_free(fmri, max_scf_fmri_size);
4212 
4213                 switch (err) {
4214                 case ENOENT:
4215                 case 0:
4216                         startd_free(restarter_fmri, max_scf_value_size);
4217                         return (-1);
4218 
4219                 case ECONNABORTED:
4220                 case ECANCELED:
4221                         startd_free(restarter_fmri, max_scf_value_size);
4222                         return (err);
4223 
4224                 default:
4225                         bad_error("libscf_get_basic_instance_data", err);
4226                 }
4227         }
4228 
4229         oldflags = v->gv_flags;
4230         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
4231             (enabled ? GV_ENBLD_NOOVR : 0);
4232 
4233         if (enabled_ovr != -1)
4234                 enabled = enabled_ovr;
4235 
4236         /*
4237          * If GV_ENBLD_NOOVR has changed, then we need to re-evaluate the
4238          * subgraph.
4239          */
4240         if (milestone > MILESTONE_NONE && v->gv_flags != oldflags)
4241                 (void) eval_subgraph(v, h);
4242 
4243         scf_instance_destroy(inst);
4244 
4245         /* Ignore restarter change for now. */
4246 
4247         startd_free(restarter_fmri, max_scf_value_size);
4248         startd_free(fmri, max_scf_fmri_size);
4249 
4250         /*
4251          * Always send _ENABLE or _DISABLE.  We could avoid this if the
4252          * restarter didn't change and the enabled value didn't change, but
4253          * that's not easy to check and improbable anyway, so we'll just do
4254          * this.
4255          */
4256         graph_enable_by_vertex(v, enabled, 1);
4257 
4258         MUTEX_UNLOCK(&dgraph_lock);
4259 
4260         return (0);
4261 }
4262 
4263 /*
4264  * Delete all of the property group dependencies of v, update inst's running
4265  * snapshot, and add the dependencies in the new snapshot.  If any of the new
4266  * dependencies would create a cycle, send _ADMIN_MAINT_ON.  Otherwise
4267  * reevaluate v's dependencies, send _START or _STOP as appropriate, and do
4268  * the same for v's dependents.
4269  *
4270  * Returns
4271  *   0 - success
4272  *   ECONNABORTED - repository connection broken
4273  *   ECANCELED - inst was deleted
4274  *   EINVAL - inst is invalid (e.g., missing general/enabled)
4275  *   -1 - libscf_snapshots_refresh() failed
4276  */
4277 static int
4278 dgraph_refresh_instance(graph_vertex_t *v, scf_instance_t *inst)
4279 {
4280         int r;
4281         int enabled;
4282         int32_t tset;
4283 
4284         assert(MUTEX_HELD(&dgraph_lock));
4285         assert(v->gv_type == GVT_INST);
4286 
4287         /* Only refresh services with valid general/enabled properties. */
4288         r = libscf_get_basic_instance_data(scf_instance_handle(inst), inst,
4289             v->gv_name, &enabled, NULL, NULL);
4290         switch (r) {
4291         case 0:
4292                 break;
4293 
4294         case ECONNABORTED:
4295         case ECANCELED:
4296                 return (r);
4297 
4298         case ENOENT:
4299                 log_framework(LOG_DEBUG,
4300                     "Ignoring %s because it has no general property group.\n",
4301                     v->gv_name);
4302                 return (EINVAL);
4303 
4304         default:
4305                 bad_error("libscf_get_basic_instance_data", r);
4306         }
4307 
4308         if ((tset = libscf_get_stn_tset(inst)) == -1) {
4309                 log_framework(LOG_WARNING,
4310                     "Failed to get notification parameters for %s: %s\n",
4311                     v->gv_name, scf_strerror(scf_error()));
4312                 tset = 0;
4313         }
4314         v->gv_stn_tset = tset;
4315         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
4316                 stn_global = tset;
4317 
4318         if (enabled == -1)
4319                 return (EINVAL);
4320 
4321         r = libscf_snapshots_refresh(inst, v->gv_name);
4322         if (r != 0) {
4323                 if (r != -1)
4324                         bad_error("libscf_snapshots_refresh", r);
4325 
4326                 /* error logged */
4327                 return (r);
4328         }
4329 
4330         r = refresh_vertex(v, inst);
4331         if (r != 0 && r != ECONNABORTED)
4332                 bad_error("refresh_vertex", r);
4333         return (r);
4334 }
4335 
4336 /*
4337  * Returns true only if none of this service's dependents are 'up' -- online
4338  * or degraded (offline is considered down in this situation). This function
4339  * is somehow similar to is_nonsubgraph_leaf() but works on subtrees.
4340  */
4341 static boolean_t
4342 insubtree_dependents_down(graph_vertex_t *v)
4343 {
4344         graph_vertex_t *vv;
4345         graph_edge_t *e;
4346 
4347         assert(MUTEX_HELD(&dgraph_lock));
4348 
4349         for (e = uu_list_first(v->gv_dependents); e != NULL;
4350             e = uu_list_next(v->gv_dependents, e)) {
4351                 vv = e->ge_vertex;
4352                 if (vv->gv_type == GVT_INST) {
4353                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4354                                 continue;
4355 
4356                         if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4357                                 continue;
4358 
4359                         if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4360                             (vv->gv_state == RESTARTER_STATE_DEGRADED))
4361                                 return (B_FALSE);
4362                 } else {
4363                         /*
4364                          * Skip all excluded dependents and decide whether
4365                          * to offline the service based on the restart_on
4366                          * on attribute.
4367                          */
4368                         if (is_depgrp_bypassed(vv))
4369                                 continue;
4370 
4371                         /*
4372                          * For dependency groups or service vertices, keep
4373                          * traversing to see if instances are running.
4374                          */
4375                         if (insubtree_dependents_down(vv) == B_FALSE)
4376                                 return (B_FALSE);
4377                 }
4378         }
4379 
4380         return (B_TRUE);
4381 }
4382 
4383 /*
4384  * Returns true only if none of this service's dependents are 'up' -- online,
4385  * degraded, or offline.
4386  */
4387 static int
4388 is_nonsubgraph_leaf(graph_vertex_t *v)
4389 {
4390         graph_vertex_t *vv;
4391         graph_edge_t *e;
4392 
4393         assert(MUTEX_HELD(&dgraph_lock));
4394 
4395         for (e = uu_list_first(v->gv_dependents);
4396             e != NULL;
4397             e = uu_list_next(v->gv_dependents, e)) {
4398 
4399                 vv = e->ge_vertex;
4400                 if (vv->gv_type == GVT_INST) {
4401                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4402                                 continue;
4403 
4404                         if (vv->gv_flags & GV_INSUBGRAPH)
4405                                 continue;
4406 
4407                         if (up_state(vv->gv_state))
4408                                 return (0);
4409                 } else {
4410                         /*
4411                          * For dependency group or service vertices, keep
4412                          * traversing to see if instances are running.
4413                          *
4414                          * We should skip exclude_all dependencies otherwise
4415                          * the vertex will never be considered as a leaf
4416                          * if the dependent is offline. The main reason for
4417                          * this is that disable_nonsubgraph_leaves() skips
4418                          * exclusion dependencies.
4419                          */
4420                         if (vv->gv_type == GVT_GROUP &&
4421                             vv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4422                                 continue;
4423 
4424                         if (!is_nonsubgraph_leaf(vv))
4425                                 return (0);
4426                 }
4427         }
4428 
4429         return (1);
4430 }
4431 
4432 /*
4433  * Disable v temporarily.  Attempt to do this by setting its enabled override
4434  * property in the repository.  If that fails, send a _DISABLE command.
4435  * Returns 0 on success and ECONNABORTED if the repository connection is
4436  * broken.
4437  */
4438 static int
4439 disable_service_temporarily(graph_vertex_t *v, scf_handle_t *h)
4440 {
4441         const char * const emsg = "Could not temporarily disable %s because "
4442             "%s.  Will stop service anyways.  Repository status for the "
4443             "service may be inaccurate.\n";
4444         const char * const emsg_cbroken =
4445             "the repository connection was broken";
4446 
4447         scf_instance_t *inst;
4448         int r;
4449 
4450         inst = scf_instance_create(h);
4451         if (inst == NULL) {
4452                 char buf[100];
4453 
4454                 (void) snprintf(buf, sizeof (buf),
4455                     "scf_instance_create() failed (%s)",
4456                     scf_strerror(scf_error()));
4457                 log_error(LOG_WARNING, emsg, v->gv_name, buf);
4458 
4459                 graph_enable_by_vertex(v, 0, 0);
4460                 return (0);
4461         }
4462 
4463         r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
4464             NULL, NULL, SCF_DECODE_FMRI_EXACT);
4465         if (r != 0) {
4466                 switch (scf_error()) {
4467                 case SCF_ERROR_CONNECTION_BROKEN:
4468                         log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4469                         graph_enable_by_vertex(v, 0, 0);
4470                         return (ECONNABORTED);
4471 
4472                 case SCF_ERROR_NOT_FOUND:
4473                         return (0);
4474 
4475                 case SCF_ERROR_HANDLE_MISMATCH:
4476                 case SCF_ERROR_INVALID_ARGUMENT:
4477                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4478                 case SCF_ERROR_NOT_BOUND:
4479                 default:
4480                         bad_error("scf_handle_decode_fmri",
4481                             scf_error());
4482                 }
4483         }
4484 
4485         r = libscf_set_enable_ovr(inst, 0);
4486         switch (r) {
4487         case 0:
4488                 scf_instance_destroy(inst);
4489                 return (0);
4490 
4491         case ECANCELED:
4492                 scf_instance_destroy(inst);
4493                 return (0);
4494 
4495         case ECONNABORTED:
4496                 log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4497                 graph_enable_by_vertex(v, 0, 0);
4498                 return (ECONNABORTED);
4499 
4500         case EPERM:
4501                 log_error(LOG_WARNING, emsg, v->gv_name,
4502                     "the repository denied permission");
4503                 graph_enable_by_vertex(v, 0, 0);
4504                 return (0);
4505 
4506         case EROFS:
4507                 log_error(LOG_WARNING, emsg, v->gv_name,
4508                     "the repository is read-only");
4509                 graph_enable_by_vertex(v, 0, 0);
4510                 return (0);
4511 
4512         default:
4513                 bad_error("libscf_set_enable_ovr", r);
4514                 /* NOTREACHED */
4515         }
4516 }
4517 
4518 /*
4519  * Of the transitive instance dependencies of v, offline those which are
4520  * in the subtree and which are leaves (i.e., have no dependents which are
4521  * "up").
4522  */
4523 void
4524 offline_subtree_leaves(graph_vertex_t *v, void *arg)
4525 {
4526         assert(MUTEX_HELD(&dgraph_lock));
4527 
4528         /* If v isn't an instance, recurse on its dependencies. */
4529         if (v->gv_type != GVT_INST) {
4530                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4531                 return;
4532         }
4533 
4534         /*
4535          * If v is not in the subtree, so should all of its dependencies,
4536          * so do nothing.
4537          */
4538         if ((v->gv_flags & GV_TOOFFLINE) == 0)
4539                 return;
4540 
4541         /* If v isn't a leaf because it's already down, recurse. */
4542         if (!up_state(v->gv_state)) {
4543                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4544                 return;
4545         }
4546 
4547         /* if v is a leaf, offline it or disable it if it's the last one */
4548         if (insubtree_dependents_down(v) == B_TRUE) {
4549                 if (v->gv_flags & GV_TODISABLE)
4550                         vertex_send_event(v,
4551                             RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
4552                 else
4553                         offline_vertex(v);
4554         }
4555 }
4556 
4557 void
4558 graph_offline_subtree_leaves(graph_vertex_t *v, void *h)
4559 {
4560         graph_walk_dependencies(v, offline_subtree_leaves, (void *)h);
4561 }
4562 
4563 
4564 /*
4565  * Of the transitive instance dependencies of v, disable those which are not
4566  * in the subgraph and which are leaves (i.e., have no dependents which are
4567  * "up").
4568  */
4569 static void
4570 disable_nonsubgraph_leaves(graph_vertex_t *v, void *arg)
4571 {
4572         assert(MUTEX_HELD(&dgraph_lock));
4573 
4574         /*
4575          * We must skip exclusion dependencies because they are allowed to
4576          * complete dependency cycles.  This is correct because A's exclusion
4577          * dependency on B doesn't bear on the order in which they should be
4578          * stopped.  Indeed, the exclusion dependency should guarantee that
4579          * they are never online at the same time.
4580          */
4581         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4582                 return;
4583 
4584         /* If v isn't an instance, recurse on its dependencies. */
4585         if (v->gv_type != GVT_INST)
4586                 goto recurse;
4587 
4588         if ((v->gv_flags & GV_CONFIGURED) == 0)
4589                 /*
4590                  * Unconfigured instances should have no dependencies, but in
4591                  * case they ever get them,
4592                  */
4593                 goto recurse;
4594 
4595         /*
4596          * If v is in the subgraph, so should all of its dependencies, so do
4597          * nothing.
4598          */
4599         if (v->gv_flags & GV_INSUBGRAPH)
4600                 return;
4601 
4602         /* If v isn't a leaf because it's already down, recurse. */
4603         if (!up_state(v->gv_state))
4604                 goto recurse;
4605 
4606         /* If v is disabled but not down yet, be patient. */
4607         if ((v->gv_flags & GV_ENABLED) == 0)
4608                 return;
4609 
4610         /* If v is a leaf, disable it. */
4611         if (is_nonsubgraph_leaf(v))
4612                 (void) disable_service_temporarily(v, (scf_handle_t *)arg);
4613 
4614         return;
4615 
4616 recurse:
4617         graph_walk_dependencies(v, disable_nonsubgraph_leaves, arg);
4618 }
4619 
4620 static int
4621 stn_restarter_state(restarter_instance_state_t rstate)
4622 {
4623         static const struct statemap {
4624                 restarter_instance_state_t restarter_state;
4625                 int scf_state;
4626         } map[] = {
4627                 { RESTARTER_STATE_UNINIT, SCF_STATE_UNINIT },
4628                 { RESTARTER_STATE_MAINT, SCF_STATE_MAINT },
4629                 { RESTARTER_STATE_OFFLINE, SCF_STATE_OFFLINE },
4630                 { RESTARTER_STATE_DISABLED, SCF_STATE_DISABLED },
4631                 { RESTARTER_STATE_ONLINE, SCF_STATE_ONLINE },
4632                 { RESTARTER_STATE_DEGRADED, SCF_STATE_DEGRADED }
4633         };
4634 
4635         int i;
4636 
4637         for (i = 0; i < sizeof (map) / sizeof (map[0]); i++) {
4638                 if (rstate == map[i].restarter_state)
4639                         return (map[i].scf_state);
4640         }
4641 
4642         return (-1);
4643 }
4644 
4645 /*
4646  * State transition counters
4647  * Not incremented atomically - indicative only
4648  */
4649 static uint64_t stev_ct_maint;
4650 static uint64_t stev_ct_hwerr;
4651 static uint64_t stev_ct_service;
4652 static uint64_t stev_ct_global;
4653 static uint64_t stev_ct_noprefs;
4654 static uint64_t stev_ct_from_uninit;
4655 static uint64_t stev_ct_bad_state;
4656 static uint64_t stev_ct_ovr_prefs;
4657 
4658 static void
4659 dgraph_state_transition_notify(graph_vertex_t *v,
4660     restarter_instance_state_t old_state, restarter_str_t reason)
4661 {
4662         restarter_instance_state_t new_state = v->gv_state;
4663         int stn_transition, maint;
4664         int from, to;
4665         nvlist_t *attr;
4666         fmev_pri_t pri = FMEV_LOPRI;
4667         int raise = 0;
4668 
4669         if ((from = stn_restarter_state(old_state)) == -1 ||
4670             (to = stn_restarter_state(new_state)) == -1) {
4671                 stev_ct_bad_state++;
4672                 return;
4673         }
4674 
4675         stn_transition = from << 16 | to;
4676 
4677         maint = (to == SCF_STATE_MAINT || from == SCF_STATE_MAINT);
4678 
4679         if (maint) {
4680                 /*
4681                  * All transitions to/from maintenance state must raise
4682                  * an event.
4683                  */
4684                 raise++;
4685                 pri = FMEV_HIPRI;
4686                 stev_ct_maint++;
4687         } else if (reason == restarter_str_ct_ev_hwerr) {
4688                 /*
4689                  * All transitions caused by hardware fault must raise
4690                  * an event
4691                  */
4692                 raise++;
4693                 pri = FMEV_HIPRI;
4694                 stev_ct_hwerr++;
4695         } else if (stn_transition & v->gv_stn_tset) {
4696                 /*
4697                  * Specifically enabled event.
4698                  */
4699                 raise++;
4700                 stev_ct_service++;
4701         } else if (from == SCF_STATE_UNINIT) {
4702                 /*
4703                  * Only raise these if specifically selected above.
4704                  */
4705                 stev_ct_from_uninit++;
4706         } else if (stn_transition & stn_global &&
4707             (IS_ENABLED(v) == 1 || to == SCF_STATE_DISABLED)) {
4708                 raise++;
4709                 stev_ct_global++;
4710         } else {
4711                 stev_ct_noprefs++;
4712         }
4713 
4714         if (info_events_all) {
4715                 stev_ct_ovr_prefs++;
4716                 raise++;
4717         }
4718         if (!raise)
4719                 return;
4720 
4721         if (nvlist_alloc(&attr, NV_UNIQUE_NAME, 0) != 0 ||
4722             nvlist_add_string(attr, "fmri", v->gv_name) != 0 ||
4723             nvlist_add_uint32(attr, "reason-version",
4724             restarter_str_version()) || nvlist_add_string(attr, "reason-short",
4725             restarter_get_str_short(reason)) != 0 ||
4726             nvlist_add_string(attr, "reason-long",
4727             restarter_get_str_long(reason)) != 0 ||
4728             nvlist_add_int32(attr, "transition", stn_transition) != 0) {
4729                 log_framework(LOG_WARNING,
4730                     "FMEV: %s could not create nvlist for transition "
4731                     "event: %s\n", v->gv_name, strerror(errno));
4732                 nvlist_free(attr);
4733                 return;
4734         }
4735 
4736         if (fmev_rspublish_nvl(FMEV_RULESET_SMF, "state-transition",
4737             instance_state_str[new_state], pri, attr) != FMEV_SUCCESS) {
4738                 log_framework(LOG_DEBUG,
4739                     "FMEV: %s failed to publish transition event: %s\n",
4740                     v->gv_name, fmev_strerror(fmev_errno));
4741                 nvlist_free(attr);
4742         }
4743 }
4744 
4745 /*
4746  * Find the vertex for inst_name.  If it doesn't exist, return ENOENT.
4747  * Otherwise set its state to state.  If the instance has entered a state
4748  * which requires automatic action, take it (Uninitialized: do
4749  * dgraph_refresh_instance() without the snapshot update.  Disabled: if the
4750  * instance should be enabled, send _ENABLE.  Offline: if the instance should
4751  * be disabled, send _DISABLE, and if its dependencies are satisfied, send
4752  * _START.  Online, Degraded: if the instance wasn't running, update its start
4753  * snapshot.  Maintenance: no action.)
4754  *
4755  * Also fails with ECONNABORTED, or EINVAL if state is invalid.
4756  */
4757 static int
4758 dgraph_set_instance_state(scf_handle_t *h, const char *inst_name,
4759     protocol_states_t *states)
4760 {
4761         graph_vertex_t *v;
4762         int err = 0;
4763         restarter_instance_state_t old_state;
4764         restarter_instance_state_t state = states->ps_state;
4765         restarter_error_t serr = states->ps_err;
4766 
4767         MUTEX_LOCK(&dgraph_lock);
4768 
4769         v = vertex_get_by_name(inst_name);
4770         if (v == NULL) {
4771                 MUTEX_UNLOCK(&dgraph_lock);
4772                 return (ENOENT);
4773         }
4774 
4775         assert(v->gv_type == GVT_INST);
4776 
4777         switch (state) {
4778         case RESTARTER_STATE_UNINIT:
4779         case RESTARTER_STATE_DISABLED:
4780         case RESTARTER_STATE_OFFLINE:
4781         case RESTARTER_STATE_ONLINE:
4782         case RESTARTER_STATE_DEGRADED:
4783         case RESTARTER_STATE_MAINT:
4784                 break;
4785 
4786         default:
4787                 MUTEX_UNLOCK(&dgraph_lock);
4788                 return (EINVAL);
4789         }
4790 
4791         log_framework(LOG_DEBUG, "Graph noting %s %s -> %s.\n", v->gv_name,
4792             instance_state_str[v->gv_state], instance_state_str[state]);
4793 
4794         old_state = v->gv_state;
4795         v->gv_state = state;
4796 
4797         v->gv_reason = states->ps_reason;
4798         err = gt_transition(h, v, serr, old_state);
4799         if (err == 0 && v->gv_state != old_state) {
4800                 dgraph_state_transition_notify(v, old_state, states->ps_reason);
4801         }
4802 
4803         MUTEX_UNLOCK(&dgraph_lock);
4804         return (err);
4805 }
4806 
4807 /*
4808  * Handle state changes during milestone shutdown.  See
4809  * dgraph_set_milestone().  If the repository connection is broken,
4810  * ECONNABORTED will be returned, though a _DISABLE command will be sent for
4811  * the vertex anyway.
4812  */
4813 int
4814 vertex_subgraph_dependencies_shutdown(scf_handle_t *h, graph_vertex_t *v,
4815     restarter_instance_state_t old_state)
4816 {
4817         int was_up, now_up;
4818         int ret = 0;
4819 
4820         assert(v->gv_type == GVT_INST);
4821 
4822         /* Don't care if we're not going to a milestone. */
4823         if (milestone == NULL)
4824                 return (0);
4825 
4826         /* Don't care if we already finished coming down. */
4827         if (non_subgraph_svcs == 0)
4828                 return (0);
4829 
4830         /* Don't care if the service is in the subgraph. */
4831         if (v->gv_flags & GV_INSUBGRAPH)
4832                 return (0);
4833 
4834         /*
4835          * Update non_subgraph_svcs.  It is the number of non-subgraph
4836          * services which are in online, degraded, or offline.
4837          */
4838 
4839         was_up = up_state(old_state);
4840         now_up = up_state(v->gv_state);
4841 
4842         if (!was_up && now_up) {
4843                 ++non_subgraph_svcs;
4844         } else if (was_up && !now_up) {
4845                 --non_subgraph_svcs;
4846 
4847                 if (non_subgraph_svcs == 0) {
4848                         if (halting != -1) {
4849                                 do_uadmin();
4850                         } else if (go_single_user_mode || go_to_level1) {
4851                                 (void) startd_thread_create(single_user_thread,
4852                                     NULL);
4853                         }
4854                         return (0);
4855                 }
4856         }
4857 
4858         /* If this service is a leaf, it should be disabled. */
4859         if ((v->gv_flags & GV_ENABLED) && is_nonsubgraph_leaf(v)) {
4860                 int r;
4861 
4862                 r = disable_service_temporarily(v, h);
4863                 switch (r) {
4864                 case 0:
4865                         break;
4866 
4867                 case ECONNABORTED:
4868                         ret = ECONNABORTED;
4869                         break;
4870 
4871                 default:
4872                         bad_error("disable_service_temporarily", r);
4873                 }
4874         }
4875 
4876         /*
4877          * If the service just came down, propagate the disable to the newly
4878          * exposed leaves.
4879          */
4880         if (was_up && !now_up)
4881                 graph_walk_dependencies(v, disable_nonsubgraph_leaves,
4882                     (void *)h);
4883 
4884         return (ret);
4885 }
4886 
4887 /*
4888  * Decide whether to start up an sulogin thread after a service is
4889  * finished changing state.  Only need to do the full can_come_up()
4890  * evaluation if an instance is changing state, we're not halfway through
4891  * loading the thread, and we aren't shutting down or going to the single
4892  * user milestone.
4893  */
4894 void
4895 graph_transition_sulogin(restarter_instance_state_t state,
4896     restarter_instance_state_t old_state)
4897 {
4898         assert(MUTEX_HELD(&dgraph_lock));
4899 
4900         if (state != old_state && st->st_load_complete &&
4901             !go_single_user_mode && !go_to_level1 &&
4902             halting == -1) {
4903                 if (!sulogin_thread_running && !can_come_up()) {
4904                         (void) startd_thread_create(sulogin_thread, NULL);
4905                         sulogin_thread_running = B_TRUE;
4906                 }
4907         }
4908 }
4909 
4910 /*
4911  * Propagate a start, stop event, or a satisfiability event.
4912  *
4913  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4914  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4915  * full dependent graph to check for newly satisfied nodes.  This is
4916  * necessary for cases when non-direct dependents may be effected but direct
4917  * dependents may not (e.g. for optional_all evaluations, see the
4918  * propagate_satbility() comments).
4919  *
4920  * PROPAGATE_SAT should be used whenever a non-running service moves into
4921  * a state which can satisfy optional dependencies, like disabled or
4922  * maintenance.
4923  */
4924 void
4925 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4926     restarter_error_t rerr)
4927 {
4928         if (type == PROPAGATE_STOP) {
4929                 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4930         } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4931                 graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
4932 
4933                 if (type == PROPAGATE_SAT)
4934                         propagate_satbility(v);
4935         } else {
4936 #ifndef NDEBUG
4937                 uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4938                     __LINE__, type);
4939 #endif
4940                 abort();
4941         }
4942 }
4943 
4944 /*
4945  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4946  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4947  * all property group dependencies, and the dependency on the restarter,
4948  * disposing of vertices as appropriate.  If other vertices depend on this
4949  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4950  * returns 0.
4951  */
4952 static int
4953 dgraph_remove_instance(const char *fmri, scf_handle_t *h)
4954 {
4955         graph_vertex_t *v;
4956         graph_edge_t *e;
4957         uu_list_t *old_deps;
4958         int err;
4959 
4960         log_framework(LOG_DEBUG, "Graph engine: Removing %s.\n", fmri);
4961 
4962         MUTEX_LOCK(&dgraph_lock);
4963 
4964         v = vertex_get_by_name(fmri);
4965         if (v == NULL) {
4966                 MUTEX_UNLOCK(&dgraph_lock);
4967                 return (0);
4968         }
4969 
4970         /* Send restarter delete event. */
4971         if (v->gv_flags & GV_CONFIGURED)
4972                 graph_unset_restarter(v);
4973 
4974         if (milestone > MILESTONE_NONE) {
4975                 /*
4976                  * Make a list of v's current dependencies so we can
4977                  * reevaluate their GV_INSUBGRAPH flags after the dependencies
4978                  * are removed.
4979                  */
4980                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
4981 
4982                 err = uu_list_walk(v->gv_dependencies,
4983                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
4984                 assert(err == 0);
4985         }
4986 
4987         delete_instance_dependencies(v, B_TRUE);
4988 
4989         /*
4990          * Deleting an instance can both satisfy and unsatisfy dependencies,
4991          * depending on their type.  First propagate the stop as a RERR_RESTART
4992          * event -- deletion isn't a fault, just a normal stop.  This gives
4993          * dependent services the chance to do a clean shutdown.  Then, mark
4994          * the service as unconfigured and propagate the start event for the
4995          * optional_all dependencies that might have become satisfied.
4996          */
4997         graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
4998 
4999         v->gv_flags &= ~GV_CONFIGURED;
5000         v->gv_flags &= ~GV_DEATHROW;
5001 
5002         graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
5003         propagate_satbility(v);
5004 
5005         /*
5006          * If there are no (non-service) dependents, the vertex can be
5007          * completely removed.
5008          */
5009         if (v != milestone && v->gv_refs == 0 &&
5010             uu_list_numnodes(v->gv_dependents) == 1)
5011                 remove_inst_vertex(v);
5012 
5013         if (milestone > MILESTONE_NONE) {
5014                 void *cookie = NULL;
5015 
5016                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5017                         v = e->ge_vertex;
5018 
5019                         if (vertex_unref(v) == VERTEX_INUSE)
5020                                 while (eval_subgraph(v, h) == ECONNABORTED)
5021                                         libscf_handle_rebind(h);
5022 
5023                         startd_free(e, sizeof (*e));
5024                 }
5025 
5026                 uu_list_destroy(old_deps);
5027         }
5028 
5029         MUTEX_UNLOCK(&dgraph_lock);
5030 
5031         return (0);
5032 }
5033 
5034 /*
5035  * Return the eventual (maybe current) milestone in the form of a
5036  * legacy runlevel.
5037  */
5038 static char
5039 target_milestone_as_runlevel()
5040 {
5041         assert(MUTEX_HELD(&dgraph_lock));
5042 
5043         if (milestone == NULL)
5044                 return ('3');
5045         else if (milestone == MILESTONE_NONE)
5046                 return ('0');
5047 
5048         if (strcmp(milestone->gv_name, multi_user_fmri) == 0)
5049                 return ('2');
5050         else if (strcmp(milestone->gv_name, single_user_fmri) == 0)
5051                 return ('S');
5052         else if (strcmp(milestone->gv_name, multi_user_svr_fmri) == 0)
5053                 return ('3');
5054 
5055 #ifndef NDEBUG
5056         (void) fprintf(stderr, "%s:%d: Unknown milestone name \"%s\".\n",
5057             __FILE__, __LINE__, milestone->gv_name);
5058 #endif
5059         abort();
5060         /* NOTREACHED */
5061 }
5062 
5063 static struct {
5064         char    rl;
5065         int     sig;
5066 } init_sigs[] = {
5067         { 'S', SIGBUS },
5068         { '0', SIGINT },
5069         { '1', SIGQUIT },
5070         { '2', SIGILL },
5071         { '3', SIGTRAP },
5072         { '4', SIGIOT },
5073         { '5', SIGEMT },
5074         { '6', SIGFPE },
5075         { 0, 0 }
5076 };
5077 
5078 static void
5079 signal_init(char rl)
5080 {
5081         pid_t init_pid;
5082         int i;
5083 
5084         assert(MUTEX_HELD(&dgraph_lock));
5085 
5086         if (zone_getattr(getzoneid(), ZONE_ATTR_INITPID, &init_pid,
5087             sizeof (init_pid)) != sizeof (init_pid)) {
5088                 log_error(LOG_NOTICE, "Could not get pid to signal init.\n");
5089                 return;
5090         }
5091 
5092         for (i = 0; init_sigs[i].rl != 0; ++i)
5093                 if (init_sigs[i].rl == rl)
5094                         break;
5095 
5096         if (init_sigs[i].rl != 0) {
5097                 if (kill(init_pid, init_sigs[i].sig) != 0) {
5098                         switch (errno) {
5099                         case EPERM:
5100                         case ESRCH:
5101                                 log_error(LOG_NOTICE, "Could not signal init: "
5102                                     "%s.\n", strerror(errno));
5103                                 break;
5104 
5105                         case EINVAL:
5106                         default:
5107                                 bad_error("kill", errno);
5108                         }
5109                 }
5110         }
5111 }
5112 
5113 /*
5114  * This is called when one of the major milestones changes state, or when
5115  * init is signalled and tells us it was told to change runlevel.  We wait
5116  * to reach the milestone because this allows /etc/inittab entries to retain
5117  * some boot ordering: historically, entries could place themselves before/after
5118  * the running of /sbin/rcX scripts but we can no longer make the
5119  * distinction because the /sbin/rcX scripts no longer exist as punctuation
5120  * marks in /etc/inittab.
5121  *
5122  * Also, we only trigger an update when we reach the eventual target
5123  * milestone: without this, an /etc/inittab entry marked only for
5124  * runlevel 2 would be executed for runlevel 3, which is not how
5125  * /etc/inittab entries work.
5126  *
5127  * If we're single user coming online, then we set utmpx to the target
5128  * runlevel so that legacy scripts can work as expected.
5129  */
5130 static void
5131 graph_runlevel_changed(char rl, int online)
5132 {
5133         char trl;
5134 
5135         assert(MUTEX_HELD(&dgraph_lock));
5136 
5137         trl = target_milestone_as_runlevel();
5138 
5139         if (online) {
5140                 if (rl == trl) {
5141                         current_runlevel = trl;
5142                         signal_init(trl);
5143                 } else if (rl == 'S') {
5144                         /*
5145                          * At boot, set the entry early for the benefit of the
5146                          * legacy init scripts.
5147                          */
5148                         utmpx_set_runlevel(trl, 'S', B_FALSE);
5149                 }
5150         } else {
5151                 if (rl == '3' && trl == '2') {
5152                         current_runlevel = trl;
5153                         signal_init(trl);
5154                 } else if (rl == '2' && trl == 'S') {
5155                         current_runlevel = trl;
5156                         signal_init(trl);
5157                 }
5158         }
5159 }
5160 
5161 /*
5162  * Move to a backwards-compatible runlevel by executing the appropriate
5163  * /etc/rc?.d/K* scripts and/or setting the milestone.
5164  *
5165  * Returns
5166  *   0 - success
5167  *   ECONNRESET - success, but handle was reset
5168  *   ECONNABORTED - repository connection broken
5169  *   ECANCELED - pg was deleted
5170  */
5171 static int
5172 dgraph_set_runlevel(scf_propertygroup_t *pg, scf_property_t *prop)
5173 {
5174         char rl;
5175         scf_handle_t *h;
5176         int r;
5177         const char *ms = NULL;  /* what to commit as options/milestone */
5178         boolean_t rebound = B_FALSE;
5179         int mark_rl = 0;
5180 
5181         const char * const stop = "stop";
5182 
5183         r = libscf_extract_runlevel(prop, &rl);
5184         switch (r) {
5185         case 0:
5186                 break;
5187 
5188         case ECONNABORTED:
5189         case ECANCELED:
5190                 return (r);
5191 
5192         case EINVAL:
5193         case ENOENT:
5194                 log_error(LOG_WARNING, "runlevel property is misconfigured; "
5195                     "ignoring.\n");
5196                 /* delete the bad property */
5197                 goto nolock_out;
5198 
5199         default:
5200                 bad_error("libscf_extract_runlevel", r);
5201         }
5202 
5203         switch (rl) {
5204         case 's':
5205                 rl = 'S';
5206                 /* FALLTHROUGH */
5207 
5208         case 'S':
5209         case '2':
5210         case '3':
5211                 /*
5212                  * These cases cause a milestone change, so
5213                  * graph_runlevel_changed() will eventually deal with
5214                  * signalling init.
5215                  */
5216                 break;
5217 
5218         case '0':
5219         case '1':
5220         case '4':
5221         case '5':
5222         case '6':
5223                 mark_rl = 1;
5224                 break;
5225 
5226         default:
5227                 log_framework(LOG_NOTICE, "Unknown runlevel '%c'.\n", rl);
5228                 ms = NULL;
5229                 goto nolock_out;
5230         }
5231 
5232         h = scf_pg_handle(pg);
5233 
5234         MUTEX_LOCK(&dgraph_lock);
5235 
5236         /*
5237          * Since this triggers no milestone changes, force it by hand.
5238          */
5239         if (current_runlevel == '4' && rl == '3')
5240                 mark_rl = 1;
5241 
5242         /*
5243          * 1. If we are here after an "init X":
5244          *
5245          * init X
5246          *      init/lscf_set_runlevel()
5247          *              process_pg_event()
5248          *              dgraph_set_runlevel()
5249          *
5250          * then we haven't passed through graph_runlevel_changed() yet,
5251          * therefore 'current_runlevel' has not changed for sure but 'rl' has.
5252          * In consequence, if 'rl' is lower than 'current_runlevel', we change
5253          * the system runlevel and execute the appropriate /etc/rc?.d/K* scripts
5254          * past this test.
5255          *
5256          * 2. On the other hand, if we are here after a "svcadm milestone":
5257          *
5258          * svcadm milestone X
5259          *      dgraph_set_milestone()
5260          *              handle_graph_update_event()
5261          *              dgraph_set_instance_state()
5262          *              graph_post_X_[online|offline]()
5263          *              graph_runlevel_changed()
5264          *              signal_init()
5265          *                      init/lscf_set_runlevel()
5266          *                              process_pg_event()
5267          *                              dgraph_set_runlevel()
5268          *
5269          * then we already passed through graph_runlevel_changed() (by the way
5270          * of dgraph_set_milestone()) and 'current_runlevel' may have changed
5271          * and already be equal to 'rl' so we are going to return immediately
5272          * from dgraph_set_runlevel() without changing the system runlevel and
5273          * without executing the /etc/rc?.d/K* scripts.
5274          */
5275         if (rl == current_runlevel) {
5276                 ms = NULL;
5277                 goto out;
5278         }
5279 
5280         log_framework(LOG_DEBUG, "Changing to runlevel '%c'.\n", rl);
5281 
5282         /*
5283          * Make sure stop rc scripts see the new settings via who -r.
5284          */
5285         utmpx_set_runlevel(rl, current_runlevel, B_TRUE);
5286 
5287         /*
5288          * Some run levels don't have a direct correspondence to any
5289          * milestones, so we have to signal init directly.
5290          */
5291         if (mark_rl) {
5292                 current_runlevel = rl;
5293                 signal_init(rl);
5294         }
5295 
5296         switch (rl) {
5297         case 'S':
5298                 uu_warn("The system is coming down for administration.  "
5299                     "Please wait.\n");
5300                 fork_rc_script(rl, stop, B_FALSE);
5301                 ms = single_user_fmri;
5302                 go_single_user_mode = B_TRUE;
5303                 break;
5304 
5305         case '0':
5306                 halting_time = time(NULL);
5307                 fork_rc_script(rl, stop, B_TRUE);
5308                 halting = AD_HALT;
5309                 goto uadmin;
5310 
5311         case '5':
5312                 halting_time = time(NULL);
5313                 fork_rc_script(rl, stop, B_TRUE);
5314                 halting = AD_POWEROFF;
5315                 goto uadmin;
5316 
5317         case '6':
5318                 halting_time = time(NULL);
5319                 fork_rc_script(rl, stop, B_TRUE);
5320                 if (scf_is_fastboot_default() && getzoneid() == GLOBAL_ZONEID)
5321                         halting = AD_FASTREBOOT;
5322                 else
5323                         halting = AD_BOOT;
5324 
5325 uadmin:
5326                 uu_warn("The system is coming down.  Please wait.\n");
5327                 ms = "none";
5328 
5329                 /*
5330                  * We can't wait until all services are offline since this
5331                  * thread is responsible for taking them offline.  Instead we
5332                  * set halting to the second argument for uadmin() and call
5333                  * do_uadmin() from dgraph_set_instance_state() when
5334                  * appropriate.
5335                  */
5336                 break;
5337 
5338         case '1':
5339                 if (current_runlevel != 'S') {
5340                         uu_warn("Changing to state 1.\n");
5341                         fork_rc_script(rl, stop, B_FALSE);
5342                 } else {
5343                         uu_warn("The system is coming up for administration.  "
5344                             "Please wait.\n");
5345                 }
5346                 ms = single_user_fmri;
5347                 go_to_level1 = B_TRUE;
5348                 break;
5349 
5350         case '2':
5351                 if (current_runlevel == '3' || current_runlevel == '4')
5352                         fork_rc_script(rl, stop, B_FALSE);
5353                 ms = multi_user_fmri;
5354                 break;
5355 
5356         case '3':
5357         case '4':
5358                 ms = "all";
5359                 break;
5360 
5361         default:
5362 #ifndef NDEBUG
5363                 (void) fprintf(stderr, "%s:%d: Uncaught case %d ('%c').\n",
5364                     __FILE__, __LINE__, rl, rl);
5365 #endif
5366                 abort();
5367         }
5368 
5369 out:
5370         MUTEX_UNLOCK(&dgraph_lock);
5371 
5372 nolock_out:
5373         switch (r = libscf_clear_runlevel(pg, ms)) {
5374         case 0:
5375                 break;
5376 
5377         case ECONNABORTED:
5378                 libscf_handle_rebind(h);
5379                 rebound = B_TRUE;
5380                 goto nolock_out;
5381 
5382         case ECANCELED:
5383                 break;
5384 
5385         case EPERM:
5386         case EACCES:
5387         case EROFS:
5388                 log_error(LOG_NOTICE, "Could not delete \"%s/%s\" property: "
5389                     "%s.\n", SCF_PG_OPTIONS, "runlevel", strerror(r));
5390                 break;
5391 
5392         default:
5393                 bad_error("libscf_clear_runlevel", r);
5394         }
5395 
5396         return (rebound ? ECONNRESET : 0);
5397 }
5398 
5399 /*
5400  * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5401  * to the instances that are supposed to go offline during an
5402  * administrative disable operation.
5403  */
5404 static int
5405 mark_subtree(graph_edge_t *e, void *arg)
5406 {
5407         graph_vertex_t *v;
5408         int r;
5409 
5410         v = e->ge_vertex;
5411 
5412         /* If it's already in the subgraph, skip. */
5413         if (v->gv_flags & GV_TOOFFLINE)
5414                 return (UU_WALK_NEXT);
5415 
5416         switch (v->gv_type) {
5417         case GVT_INST:
5418                 /* If the instance is already disabled, skip it. */
5419                 if (!(v->gv_flags & GV_ENABLED))
5420                         return (UU_WALK_NEXT);
5421 
5422                 v->gv_flags |= GV_TOOFFLINE;
5423                 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5424                 break;
5425         case GVT_GROUP:
5426                 /*
5427                  * Skip all excluded dependents and decide whether to offline
5428                  * the service based on the restart_on attribute.
5429                  */
5430                 if (is_depgrp_bypassed(v))
5431                         return (UU_WALK_NEXT);
5432                 break;
5433         }
5434 
5435         r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5436             0);
5437         assert(r == 0);
5438         return (UU_WALK_NEXT);
5439 }
5440 
5441 static int
5442 mark_subgraph(graph_edge_t *e, void *arg)
5443 {
5444         graph_vertex_t *v;
5445         int r;
5446         int optional = (int)arg;
5447 
5448         v = e->ge_vertex;
5449 
5450         /* If it's already in the subgraph, skip. */
5451         if (v->gv_flags & GV_INSUBGRAPH)
5452                 return (UU_WALK_NEXT);
5453 
5454         /*
5455          * Keep track if walk has entered an optional dependency group
5456          */
5457         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_OPTIONAL_ALL) {
5458                 optional = 1;
5459         }
5460         /*
5461          * Quit if we are in an optional dependency group and the instance
5462          * is disabled
5463          */
5464         if (optional && (v->gv_type == GVT_INST) &&
5465             (!(v->gv_flags & GV_ENBLD_NOOVR)))
5466                 return (UU_WALK_NEXT);
5467 
5468         v->gv_flags |= GV_INSUBGRAPH;
5469 
5470         /* Skip all excluded dependencies. */
5471         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
5472                 return (UU_WALK_NEXT);
5473 
5474         r = uu_list_walk(v->gv_dependencies, (uu_walk_fn_t *)mark_subgraph,
5475             (void *)optional, 0);
5476         assert(r == 0);
5477         return (UU_WALK_NEXT);
5478 }
5479 
5480 /*
5481  * Bring down all services which are not dependencies of fmri.  The
5482  * dependencies of fmri (direct & indirect) will constitute the "subgraph",
5483  * and will have the GV_INSUBGRAPH flag set.  The rest must be brought down,
5484  * which means the state is "disabled", "maintenance", or "uninitialized".  We
5485  * could consider "offline" to be down, and refrain from sending start
5486  * commands for such services, but that's not strictly necessary, so we'll
5487  * decline to intrude on the state machine.  It would probably confuse users
5488  * anyway.
5489  *
5490  * The services should be brought down in reverse-dependency order, so we
5491  * can't do it all at once here.  We initiate by override-disabling the leaves
5492  * of the dependency tree -- those services which are up but have no
5493  * dependents which are up.  When they come down,
5494  * vertex_subgraph_dependencies_shutdown() will override-disable the newly
5495  * exposed leaves.  Perseverance will ensure completion.
5496  *
5497  * Sometimes we need to take action when the transition is complete, like
5498  * start sulogin or halt the system.  To tell when we're done, we initialize
5499  * non_subgraph_svcs here to be the number of services which need to come
5500  * down.  As each does, we decrement the counter.  When it hits zero, we take
5501  * the appropriate action.  See vertex_subgraph_dependencies_shutdown().
5502  *
5503  * In case we're coming up, we also remove any enable-overrides for the
5504  * services which are dependencies of fmri.
5505  *
5506  * If norepository is true, the function will not change the repository.
5507  *
5508  * The decision to change the system run level in accordance with the milestone
5509  * is taken in dgraph_set_runlevel().
5510  *
5511  * Returns
5512  *   0 - success
5513  *   ECONNRESET - success, but handle was rebound
5514  *   EINVAL - fmri is invalid (error is logged)
5515  *   EALREADY - the milestone is already set to fmri
5516  *   ENOENT - a configured vertex does not exist for fmri (an error is logged)
5517  */
5518 static int
5519 dgraph_set_milestone(const char *fmri, scf_handle_t *h, boolean_t norepository)
5520 {
5521         const char *cfmri, *fs;
5522         graph_vertex_t *nm, *v;
5523         int ret = 0, r;
5524         scf_instance_t *inst;
5525         boolean_t isall, isnone, rebound = B_FALSE;
5526 
5527         /* Validate fmri */
5528         isall = (strcmp(fmri, "all") == 0);
5529         isnone = (strcmp(fmri, "none") == 0);
5530 
5531         if (!isall && !isnone) {
5532                 if (fmri_canonify(fmri, (char **)&cfmri, B_FALSE) == EINVAL)
5533                         goto reject;
5534 
5535                 if (strcmp(cfmri, single_user_fmri) != 0 &&
5536                     strcmp(cfmri, multi_user_fmri) != 0 &&
5537                     strcmp(cfmri, multi_user_svr_fmri) != 0) {
5538                         startd_free((void *)cfmri, max_scf_fmri_size);
5539 reject:
5540                         log_framework(LOG_WARNING,
5541                             "Rejecting request for invalid milestone \"%s\".\n",
5542                             fmri);
5543                         return (EINVAL);
5544                 }
5545         }
5546 
5547         inst = safe_scf_instance_create(h);
5548 
5549         MUTEX_LOCK(&dgraph_lock);
5550 
5551         if (milestone == NULL) {
5552                 if (isall) {
5553                         log_framework(LOG_DEBUG,
5554                             "Milestone already set to all.\n");
5555                         ret = EALREADY;
5556                         goto out;
5557                 }
5558         } else if (milestone == MILESTONE_NONE) {
5559                 if (isnone) {
5560                         log_framework(LOG_DEBUG,
5561                             "Milestone already set to none.\n");
5562                         ret = EALREADY;
5563                         goto out;
5564                 }
5565         } else {
5566                 if (!isall && !isnone &&
5567                     strcmp(cfmri, milestone->gv_name) == 0) {
5568                         log_framework(LOG_DEBUG,
5569                             "Milestone already set to %s.\n", cfmri);
5570                         ret = EALREADY;
5571                         goto out;
5572                 }
5573         }
5574 
5575         if (!isall && !isnone) {
5576                 nm = vertex_get_by_name(cfmri);
5577                 if (nm == NULL || !(nm->gv_flags & GV_CONFIGURED)) {
5578                         log_framework(LOG_WARNING, "Cannot set milestone to %s "
5579                             "because no such service exists.\n", cfmri);
5580                         ret = ENOENT;
5581                         goto out;
5582                 }
5583         }
5584 
5585         log_framework(LOG_DEBUG, "Changing milestone to %s.\n", fmri);
5586 
5587         /*
5588          * Set milestone, removing the old one if this was the last reference.
5589          */
5590         if (milestone > MILESTONE_NONE)
5591                 (void) vertex_unref(milestone);
5592 
5593         if (isall)
5594                 milestone = NULL;
5595         else if (isnone)
5596                 milestone = MILESTONE_NONE;
5597         else {
5598                 milestone = nm;
5599                 /* milestone should count as a reference */
5600                 vertex_ref(milestone);
5601         }
5602 
5603         /* Clear all GV_INSUBGRAPH bits. */
5604         for (v = uu_list_first(dgraph); v != NULL; v = uu_list_next(dgraph, v))
5605                 v->gv_flags &= ~GV_INSUBGRAPH;
5606 
5607         if (!isall && !isnone) {
5608                 /* Set GV_INSUBGRAPH for milestone & descendents. */
5609                 milestone->gv_flags |= GV_INSUBGRAPH;
5610 
5611                 r = uu_list_walk(milestone->gv_dependencies,
5612                     (uu_walk_fn_t *)mark_subgraph, NULL, 0);
5613                 assert(r == 0);
5614         }
5615 
5616         /* Un-override services in the subgraph & override-disable the rest. */
5617         if (norepository)
5618                 goto out;
5619 
5620         non_subgraph_svcs = 0;
5621         for (v = uu_list_first(dgraph);
5622             v != NULL;
5623             v = uu_list_next(dgraph, v)) {
5624                 if (v->gv_type != GVT_INST ||
5625                     (v->gv_flags & GV_CONFIGURED) == 0)
5626                         continue;
5627 
5628 again:
5629                 r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
5630                     NULL, NULL, SCF_DECODE_FMRI_EXACT);
5631                 if (r != 0) {
5632                         switch (scf_error()) {
5633                         case SCF_ERROR_CONNECTION_BROKEN:
5634                         default:
5635                                 libscf_handle_rebind(h);
5636                                 rebound = B_TRUE;
5637                                 goto again;
5638 
5639                         case SCF_ERROR_NOT_FOUND:
5640                                 continue;
5641 
5642                         case SCF_ERROR_HANDLE_MISMATCH:
5643                         case SCF_ERROR_INVALID_ARGUMENT:
5644                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5645                         case SCF_ERROR_NOT_BOUND:
5646                                 bad_error("scf_handle_decode_fmri",
5647                                     scf_error());
5648                         }
5649                 }
5650 
5651                 if (isall || (v->gv_flags & GV_INSUBGRAPH)) {
5652                         r = libscf_delete_enable_ovr(inst);
5653                         fs = "libscf_delete_enable_ovr";
5654                 } else {
5655                         assert(isnone || (v->gv_flags & GV_INSUBGRAPH) == 0);
5656 
5657                         /*
5658                          * Services which are up need to come down before
5659                          * we're done, but we can only disable the leaves
5660                          * here.
5661                          */
5662 
5663                         if (up_state(v->gv_state))
5664                                 ++non_subgraph_svcs;
5665 
5666                         /* If it's already disabled, don't bother. */
5667                         if ((v->gv_flags & GV_ENABLED) == 0)
5668                                 continue;
5669 
5670                         if (!is_nonsubgraph_leaf(v))
5671                                 continue;
5672 
5673                         r = libscf_set_enable_ovr(inst, 0);
5674                         fs = "libscf_set_enable_ovr";
5675                 }
5676                 switch (r) {
5677                 case 0:
5678                 case ECANCELED:
5679                         break;
5680 
5681                 case ECONNABORTED:
5682                         libscf_handle_rebind(h);
5683                         rebound = B_TRUE;
5684                         goto again;
5685 
5686                 case EPERM:
5687                 case EROFS:
5688                         log_error(LOG_WARNING,
5689                             "Could not set %s/%s for %s: %s.\n",
5690                             SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
5691                             v->gv_name, strerror(r));
5692                         break;
5693 
5694                 default:
5695                         bad_error(fs, r);
5696                 }
5697         }
5698 
5699         if (halting != -1) {
5700                 if (non_subgraph_svcs > 1)
5701                         uu_warn("%d system services are now being stopped.\n",
5702                             non_subgraph_svcs);
5703                 else if (non_subgraph_svcs == 1)
5704                         uu_warn("One system service is now being stopped.\n");
5705                 else if (non_subgraph_svcs == 0)
5706                         do_uadmin();
5707         }
5708 
5709         ret = rebound ? ECONNRESET : 0;
5710 
5711 out:
5712         MUTEX_UNLOCK(&dgraph_lock);
5713         if (!isall && !isnone)
5714                 startd_free((void *)cfmri, max_scf_fmri_size);
5715         scf_instance_destroy(inst);
5716         return (ret);
5717 }
5718 
5719 
5720 /*
5721  * Returns 0, ECONNABORTED, or EINVAL.
5722  */
5723 static int
5724 handle_graph_update_event(scf_handle_t *h, graph_protocol_event_t *e)
5725 {
5726         int r;
5727 
5728         switch (e->gpe_type) {
5729         case GRAPH_UPDATE_RELOAD_GRAPH:
5730                 log_error(LOG_WARNING,
5731                     "graph_event: reload graph unimplemented\n");
5732                 break;
5733 
5734         case GRAPH_UPDATE_STATE_CHANGE: {
5735                 protocol_states_t *states = e->gpe_data;
5736 
5737                 switch (r = dgraph_set_instance_state(h, e->gpe_inst, states)) {
5738                 case 0:
5739                 case ENOENT:
5740                         break;
5741 
5742                 case ECONNABORTED:
5743                         return (ECONNABORTED);
5744 
5745                 case EINVAL:
5746                 default:
5747 #ifndef NDEBUG
5748                         (void) fprintf(stderr, "dgraph_set_instance_state() "
5749                             "failed with unexpected error %d at %s:%d.\n", r,
5750                             __FILE__, __LINE__);
5751 #endif
5752                         abort();
5753                 }
5754 
5755                 startd_free(states, sizeof (protocol_states_t));
5756                 break;
5757         }
5758 
5759         default:
5760                 log_error(LOG_WARNING,
5761                     "graph_event_loop received an unknown event: %d\n",
5762                     e->gpe_type);
5763                 break;
5764         }
5765 
5766         return (0);
5767 }
5768 
5769 /*
5770  * graph_event_thread()
5771  *    Wait for state changes from the restarters.
5772  */
5773 /*ARGSUSED*/
5774 void *
5775 graph_event_thread(void *unused)
5776 {
5777         scf_handle_t *h;
5778         int err;
5779 
5780         h = libscf_handle_create_bound_loop();
5781 
5782         /*CONSTCOND*/
5783         while (1) {
5784                 graph_protocol_event_t *e;
5785 
5786                 MUTEX_LOCK(&gu->gu_lock);
5787 
5788                 while (gu->gu_wakeup == 0)
5789                         (void) pthread_cond_wait(&gu->gu_cv, &gu->gu_lock);
5790 
5791                 gu->gu_wakeup = 0;
5792 
5793                 while ((e = graph_event_dequeue()) != NULL) {
5794                         MUTEX_LOCK(&e->gpe_lock);
5795                         MUTEX_UNLOCK(&gu->gu_lock);
5796 
5797                         while ((err = handle_graph_update_event(h, e)) ==
5798                             ECONNABORTED)
5799                                 libscf_handle_rebind(h);
5800 
5801                         if (err == 0)
5802                                 graph_event_release(e);
5803                         else
5804                                 graph_event_requeue(e);
5805 
5806                         MUTEX_LOCK(&gu->gu_lock);
5807                 }
5808 
5809                 MUTEX_UNLOCK(&gu->gu_lock);
5810         }
5811 
5812         /*
5813          * Unreachable for now -- there's currently no graceful cleanup
5814          * called on exit().
5815          */
5816         MUTEX_UNLOCK(&gu->gu_lock);
5817         scf_handle_destroy(h);
5818         return (NULL);
5819 }
5820 
5821 static void
5822 set_initial_milestone(scf_handle_t *h)
5823 {
5824         scf_instance_t *inst;
5825         char *fmri, *cfmri;
5826         size_t sz;
5827         int r;
5828 
5829         inst = safe_scf_instance_create(h);
5830         fmri = startd_alloc(max_scf_fmri_size);
5831 
5832         /*
5833          * If -m milestone= was specified, we want to set options_ovr/milestone
5834          * to it.  Otherwise we want to read what the milestone should be set
5835          * to.  Either way we need our inst.
5836          */
5837 get_self:
5838         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
5839             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5840                 switch (scf_error()) {
5841                 case SCF_ERROR_CONNECTION_BROKEN:
5842                         libscf_handle_rebind(h);
5843                         goto get_self;
5844 
5845                 case SCF_ERROR_NOT_FOUND:
5846                         if (st->st_subgraph != NULL &&
5847                             st->st_subgraph[0] != '\0') {
5848                                 sz = strlcpy(fmri, st->st_subgraph,
5849                                     max_scf_fmri_size);
5850                                 assert(sz < max_scf_fmri_size);
5851                         } else {
5852                                 fmri[0] = '\0';
5853                         }
5854                         break;
5855 
5856                 case SCF_ERROR_INVALID_ARGUMENT:
5857                 case SCF_ERROR_CONSTRAINT_VIOLATED:
5858                 case SCF_ERROR_HANDLE_MISMATCH:
5859                 default:
5860                         bad_error("scf_handle_decode_fmri", scf_error());
5861                 }
5862         } else {
5863                 if (st->st_subgraph != NULL && st->st_subgraph[0] != '\0') {
5864                         scf_propertygroup_t *pg;
5865 
5866                         pg = safe_scf_pg_create(h);
5867 
5868                         sz = strlcpy(fmri, st->st_subgraph, max_scf_fmri_size);
5869                         assert(sz < max_scf_fmri_size);
5870 
5871                         r = libscf_inst_get_or_add_pg(inst, SCF_PG_OPTIONS_OVR,
5872                             SCF_PG_OPTIONS_OVR_TYPE, SCF_PG_OPTIONS_OVR_FLAGS,
5873                             pg);
5874                         switch (r) {
5875                         case 0:
5876                                 break;
5877 
5878                         case ECONNABORTED:
5879                                 libscf_handle_rebind(h);
5880                                 goto get_self;
5881 
5882                         case EPERM:
5883                         case EACCES:
5884                         case EROFS:
5885                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5886                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5887                                     SCF_PROPERTY_MILESTONE, strerror(r));
5888                                 /* FALLTHROUGH */
5889 
5890                         case ECANCELED:
5891                                 sz = strlcpy(fmri, st->st_subgraph,
5892                                     max_scf_fmri_size);
5893                                 assert(sz < max_scf_fmri_size);
5894                                 break;
5895 
5896                         default:
5897                                 bad_error("libscf_inst_get_or_add_pg", r);
5898                         }
5899 
5900                         r = libscf_clear_runlevel(pg, fmri);
5901                         switch (r) {
5902                         case 0:
5903                                 break;
5904 
5905                         case ECONNABORTED:
5906                                 libscf_handle_rebind(h);
5907                                 goto get_self;
5908 
5909                         case EPERM:
5910                         case EACCES:
5911                         case EROFS:
5912                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5913                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5914                                     SCF_PROPERTY_MILESTONE, strerror(r));
5915                                 /* FALLTHROUGH */
5916 
5917                         case ECANCELED:
5918                                 sz = strlcpy(fmri, st->st_subgraph,
5919                                     max_scf_fmri_size);
5920                                 assert(sz < max_scf_fmri_size);
5921                                 break;
5922 
5923                         default:
5924                                 bad_error("libscf_clear_runlevel", r);
5925                         }
5926 
5927                         scf_pg_destroy(pg);
5928                 } else {
5929                         scf_property_t *prop;
5930                         scf_value_t *val;
5931 
5932                         prop = safe_scf_property_create(h);
5933                         val = safe_scf_value_create(h);
5934 
5935                         r = libscf_get_milestone(inst, prop, val, fmri,
5936                             max_scf_fmri_size);
5937                         switch (r) {
5938                         case 0:
5939                                 break;
5940 
5941                         case ECONNABORTED:
5942                                 libscf_handle_rebind(h);
5943                                 goto get_self;
5944 
5945                         case EINVAL:
5946                                 log_error(LOG_WARNING, "Milestone property is "
5947                                     "misconfigured.  Defaulting to \"all\".\n");
5948                                 /* FALLTHROUGH */
5949 
5950                         case ECANCELED:
5951                         case ENOENT:
5952                                 fmri[0] = '\0';
5953                                 break;
5954 
5955                         default:
5956                                 bad_error("libscf_get_milestone", r);
5957                         }
5958 
5959                         scf_value_destroy(val);
5960                         scf_property_destroy(prop);
5961                 }
5962         }
5963 
5964         if (fmri[0] == '\0' || strcmp(fmri, "all") == 0)
5965                 goto out;
5966 
5967         if (strcmp(fmri, "none") != 0) {
5968 retry:
5969                 if (scf_handle_decode_fmri(h, fmri, NULL, NULL, inst, NULL,
5970                     NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5971                         switch (scf_error()) {
5972                         case SCF_ERROR_INVALID_ARGUMENT:
5973                                 log_error(LOG_WARNING,
5974                                     "Requested milestone \"%s\" is invalid.  "
5975                                     "Reverting to \"all\".\n", fmri);
5976                                 goto out;
5977 
5978                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5979                                 log_error(LOG_WARNING, "Requested milestone "
5980                                     "\"%s\" does not specify an instance.  "
5981                                     "Reverting to \"all\".\n", fmri);
5982                                 goto out;
5983 
5984                         case SCF_ERROR_CONNECTION_BROKEN:
5985                                 libscf_handle_rebind(h);
5986                                 goto retry;
5987 
5988                         case SCF_ERROR_NOT_FOUND:
5989                                 log_error(LOG_WARNING, "Requested milestone "
5990                                     "\"%s\" not in repository.  Reverting to "
5991                                     "\"all\".\n", fmri);
5992                                 goto out;
5993 
5994                         case SCF_ERROR_HANDLE_MISMATCH:
5995                         default:
5996                                 bad_error("scf_handle_decode_fmri",
5997                                     scf_error());
5998                         }
5999                 }
6000 
6001                 r = fmri_canonify(fmri, &cfmri, B_FALSE);
6002                 assert(r == 0);
6003 
6004                 r = dgraph_add_instance(cfmri, inst, B_TRUE);
6005                 startd_free(cfmri, max_scf_fmri_size);
6006                 switch (r) {
6007                 case 0:
6008                         break;
6009 
6010                 case ECONNABORTED:
6011                         goto retry;
6012 
6013                 case EINVAL:
6014                         log_error(LOG_WARNING,
6015                             "Requested milestone \"%s\" is invalid.  "
6016                             "Reverting to \"all\".\n", fmri);
6017                         goto out;
6018 
6019                 case ECANCELED:
6020                         log_error(LOG_WARNING,
6021                             "Requested milestone \"%s\" not "
6022                             "in repository.  Reverting to \"all\".\n",
6023                             fmri);
6024                         goto out;
6025 
6026                 case EEXIST:
6027                 default:
6028                         bad_error("dgraph_add_instance", r);
6029                 }
6030         }
6031 
6032         log_console(LOG_INFO, "Booting to milestone \"%s\".\n", fmri);
6033 
6034         r = dgraph_set_milestone(fmri, h, B_FALSE);
6035         switch (r) {
6036         case 0:
6037         case ECONNRESET:
6038         case EALREADY:
6039                 break;
6040 
6041         case EINVAL:
6042         case ENOENT:
6043         default:
6044                 bad_error("dgraph_set_milestone", r);
6045         }
6046 
6047 out:
6048         startd_free(fmri, max_scf_fmri_size);
6049         scf_instance_destroy(inst);
6050 }
6051 
6052 void
6053 set_restart_milestone(scf_handle_t *h)
6054 {
6055         scf_instance_t *inst;
6056         scf_property_t *prop;
6057         scf_value_t *val;
6058         char *fmri;
6059         int r;
6060 
6061         inst = safe_scf_instance_create(h);
6062 
6063 get_self:
6064         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL,
6065             inst, NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
6066                 switch (scf_error()) {
6067                 case SCF_ERROR_CONNECTION_BROKEN:
6068                         libscf_handle_rebind(h);
6069                         goto get_self;
6070 
6071                 case SCF_ERROR_NOT_FOUND:
6072                         break;
6073 
6074                 case SCF_ERROR_INVALID_ARGUMENT:
6075                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6076                 case SCF_ERROR_HANDLE_MISMATCH:
6077                 default:
6078                         bad_error("scf_handle_decode_fmri", scf_error());
6079                 }
6080 
6081                 scf_instance_destroy(inst);
6082                 return;
6083         }
6084 
6085         prop = safe_scf_property_create(h);
6086         val = safe_scf_value_create(h);
6087         fmri = startd_alloc(max_scf_fmri_size);
6088 
6089         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6090         switch (r) {
6091         case 0:
6092                 break;
6093 
6094         case ECONNABORTED:
6095                 libscf_handle_rebind(h);
6096                 goto get_self;
6097 
6098         case ECANCELED:
6099         case ENOENT:
6100         case EINVAL:
6101                 goto out;
6102 
6103         default:
6104                 bad_error("libscf_get_milestone", r);
6105         }
6106 
6107         r = dgraph_set_milestone(fmri, h, B_TRUE);
6108         switch (r) {
6109         case 0:
6110         case ECONNRESET:
6111         case EALREADY:
6112         case EINVAL:
6113         case ENOENT:
6114                 break;
6115 
6116         default:
6117                 bad_error("dgraph_set_milestone", r);
6118         }
6119 
6120 out:
6121         startd_free(fmri, max_scf_fmri_size);
6122         scf_value_destroy(val);
6123         scf_property_destroy(prop);
6124         scf_instance_destroy(inst);
6125 }
6126 
6127 /*
6128  * void *graph_thread(void *)
6129  *
6130  * Graph management thread.
6131  */
6132 /*ARGSUSED*/
6133 void *
6134 graph_thread(void *arg)
6135 {
6136         scf_handle_t *h;
6137         int err;
6138 
6139         h = libscf_handle_create_bound_loop();
6140 
6141         if (st->st_initial)
6142                 set_initial_milestone(h);
6143 
6144         MUTEX_LOCK(&dgraph_lock);
6145         initial_milestone_set = B_TRUE;
6146         err = pthread_cond_broadcast(&initial_milestone_cv);
6147         assert(err == 0);
6148         MUTEX_UNLOCK(&dgraph_lock);
6149 
6150         libscf_populate_graph(h);
6151 
6152         if (!st->st_initial)
6153                 set_restart_milestone(h);
6154 
6155         MUTEX_LOCK(&st->st_load_lock);
6156         st->st_load_complete = 1;
6157         (void) pthread_cond_broadcast(&st->st_load_cv);
6158         MUTEX_UNLOCK(&st->st_load_lock);
6159 
6160         MUTEX_LOCK(&dgraph_lock);
6161         /*
6162          * Now that we've set st_load_complete we need to check can_come_up()
6163          * since if we booted to a milestone, then there won't be any more
6164          * state updates.
6165          */
6166         if (!go_single_user_mode && !go_to_level1 &&
6167             halting == -1) {
6168                 if (!sulogin_thread_running && !can_come_up()) {
6169                         (void) startd_thread_create(sulogin_thread, NULL);
6170                         sulogin_thread_running = B_TRUE;
6171                 }
6172         }
6173         MUTEX_UNLOCK(&dgraph_lock);
6174 
6175         (void) pthread_mutex_lock(&gu->gu_freeze_lock);
6176 
6177         /*CONSTCOND*/
6178         while (1) {
6179                 (void) pthread_cond_wait(&gu->gu_freeze_cv,
6180                     &gu->gu_freeze_lock);
6181         }
6182 
6183         /*
6184          * Unreachable for now -- there's currently no graceful cleanup
6185          * called on exit().
6186          */
6187         (void) pthread_mutex_unlock(&gu->gu_freeze_lock);
6188         scf_handle_destroy(h);
6189 
6190         return (NULL);
6191 }
6192 
6193 
6194 /*
6195  * int next_action()
6196  *   Given an array of timestamps 'a' with 'num' elements, find the
6197  *   lowest non-zero timestamp and return its index. If there are no
6198  *   non-zero elements, return -1.
6199  */
6200 static int
6201 next_action(hrtime_t *a, int num)
6202 {
6203         hrtime_t t = 0;
6204         int i = 0, smallest = -1;
6205 
6206         for (i = 0; i < num; i++) {
6207                 if (t == 0) {
6208                         t = a[i];
6209                         smallest = i;
6210                 } else if (a[i] != 0 && a[i] < t) {
6211                         t = a[i];
6212                         smallest = i;
6213                 }
6214         }
6215 
6216         if (t == 0)
6217                 return (-1);
6218         else
6219                 return (smallest);
6220 }
6221 
6222 /*
6223  * void process_actions()
6224  *   Process actions requested by the administrator. Possibilities include:
6225  *   refresh, restart, maintenance mode off, maintenance mode on,
6226  *   maintenance mode immediate, and degraded.
6227  *
6228  *   The set of pending actions is represented in the repository as a
6229  *   per-instance property group, with each action being a single property
6230  *   in that group.  This property group is converted to an array, with each
6231  *   action type having an array slot.  The actions in the array at the
6232  *   time process_actions() is called are acted on in the order of the
6233  *   timestamp (which is the value stored in the slot).  A value of zero
6234  *   indicates that there is no pending action of the type associated with
6235  *   a particular slot.
6236  *
6237  *   Sending an action event multiple times before the restarter has a
6238  *   chance to process that action will force it to be run at the last
6239  *   timestamp where it appears in the ordering.
6240  *
6241  *   Turning maintenance mode on trumps all other actions.
6242  *
6243  *   Returns 0 or ECONNABORTED.
6244  */
6245 static int
6246 process_actions(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst)
6247 {
6248         scf_property_t *prop = NULL;
6249         scf_value_t *val = NULL;
6250         scf_type_t type;
6251         graph_vertex_t *vertex;
6252         admin_action_t a;
6253         int i, ret = 0, r;
6254         hrtime_t action_ts[NACTIONS];
6255         char *inst_name;
6256 
6257         r = libscf_instance_get_fmri(inst, &inst_name);
6258         switch (r) {
6259         case 0:
6260                 break;
6261 
6262         case ECONNABORTED:
6263                 return (ECONNABORTED);
6264 
6265         case ECANCELED:
6266                 return (0);
6267 
6268         default:
6269                 bad_error("libscf_instance_get_fmri", r);
6270         }
6271 
6272         MUTEX_LOCK(&dgraph_lock);
6273 
6274         vertex = vertex_get_by_name(inst_name);
6275         if (vertex == NULL) {
6276                 MUTEX_UNLOCK(&dgraph_lock);
6277                 log_framework(LOG_DEBUG, "%s: Can't find graph vertex. "
6278                     "The instance must have been removed.\n", inst_name);
6279                 startd_free(inst_name, max_scf_fmri_size);
6280                 return (0);
6281         }
6282 
6283         prop = safe_scf_property_create(h);
6284         val = safe_scf_value_create(h);
6285 
6286         for (i = 0; i < NACTIONS; i++) {
6287                 if (scf_pg_get_property(pg, admin_actions[i], prop) != 0) {
6288                         switch (scf_error()) {
6289                         case SCF_ERROR_CONNECTION_BROKEN:
6290                         default:
6291                                 ret = ECONNABORTED;
6292                                 goto out;
6293 
6294                         case SCF_ERROR_DELETED:
6295                                 goto out;
6296 
6297                         case SCF_ERROR_NOT_FOUND:
6298                                 action_ts[i] = 0;
6299                                 continue;
6300 
6301                         case SCF_ERROR_HANDLE_MISMATCH:
6302                         case SCF_ERROR_INVALID_ARGUMENT:
6303                         case SCF_ERROR_NOT_SET:
6304                                 bad_error("scf_pg_get_property", scf_error());
6305                         }
6306                 }
6307 
6308                 if (scf_property_type(prop, &type) != 0) {
6309                         switch (scf_error()) {
6310                         case SCF_ERROR_CONNECTION_BROKEN:
6311                         default:
6312                                 ret = ECONNABORTED;
6313                                 goto out;
6314 
6315                         case SCF_ERROR_DELETED:
6316                                 action_ts[i] = 0;
6317                                 continue;
6318 
6319                         case SCF_ERROR_NOT_SET:
6320                                 bad_error("scf_property_type", scf_error());
6321                         }
6322                 }
6323 
6324                 if (type != SCF_TYPE_INTEGER) {
6325                         action_ts[i] = 0;
6326                         continue;
6327                 }
6328 
6329                 if (scf_property_get_value(prop, val) != 0) {
6330                         switch (scf_error()) {
6331                         case SCF_ERROR_CONNECTION_BROKEN:
6332                         default:
6333                                 ret = ECONNABORTED;
6334                                 goto out;
6335 
6336                         case SCF_ERROR_DELETED:
6337                                 goto out;
6338 
6339                         case SCF_ERROR_NOT_FOUND:
6340                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6341                                 action_ts[i] = 0;
6342                                 continue;
6343 
6344                         case SCF_ERROR_NOT_SET:
6345                         case SCF_ERROR_PERMISSION_DENIED:
6346                                 bad_error("scf_property_get_value",
6347                                     scf_error());
6348                         }
6349                 }
6350 
6351                 r = scf_value_get_integer(val, &action_ts[i]);
6352                 assert(r == 0);
6353         }
6354 
6355         a = ADMIN_EVENT_MAINT_ON_IMMEDIATE;
6356         if (action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ||
6357             action_ts[ADMIN_EVENT_MAINT_ON]) {
6358                 a = action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ?
6359                     ADMIN_EVENT_MAINT_ON_IMMEDIATE : ADMIN_EVENT_MAINT_ON;
6360 
6361                 vertex_send_event(vertex, admin_events[a]);
6362                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6363                 switch (r) {
6364                 case 0:
6365                 case EACCES:
6366                         break;
6367 
6368                 case ECONNABORTED:
6369                         ret = ECONNABORTED;
6370                         goto out;
6371 
6372                 case EPERM:
6373                         uu_die("Insufficient privilege.\n");
6374                         /* NOTREACHED */
6375 
6376                 default:
6377                         bad_error("libscf_unset_action", r);
6378                 }
6379         }
6380 
6381         while ((a = next_action(action_ts, NACTIONS)) != -1) {
6382                 log_framework(LOG_DEBUG,
6383                     "Graph: processing %s action for %s.\n", admin_actions[a],
6384                     inst_name);
6385 
6386                 if (a == ADMIN_EVENT_REFRESH) {
6387                         r = dgraph_refresh_instance(vertex, inst);
6388                         switch (r) {
6389                         case 0:
6390                         case ECANCELED:
6391                         case EINVAL:
6392                         case -1:
6393                                 break;
6394 
6395                         case ECONNABORTED:
6396                                 /* pg & inst are reset now, so just return. */
6397                                 ret = ECONNABORTED;
6398                                 goto out;
6399 
6400                         default:
6401                                 bad_error("dgraph_refresh_instance", r);
6402                         }
6403                 }
6404 
6405                 vertex_send_event(vertex, admin_events[a]);
6406 
6407                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6408                 switch (r) {
6409                 case 0:
6410                 case EACCES:
6411                         break;
6412 
6413                 case ECONNABORTED:
6414                         ret = ECONNABORTED;
6415                         goto out;
6416 
6417                 case EPERM:
6418                         uu_die("Insufficient privilege.\n");
6419                         /* NOTREACHED */
6420 
6421                 default:
6422                         bad_error("libscf_unset_action", r);
6423                 }
6424 
6425                 action_ts[a] = 0;
6426         }
6427 
6428 out:
6429         MUTEX_UNLOCK(&dgraph_lock);
6430 
6431         scf_property_destroy(prop);
6432         scf_value_destroy(val);
6433         startd_free(inst_name, max_scf_fmri_size);
6434         return (ret);
6435 }
6436 
6437 /*
6438  * inst and pg_name are scratch space, and are unset on entry.
6439  * Returns
6440  *   0 - success
6441  *   ECONNRESET - success, but repository handle rebound
6442  *   ECONNABORTED - repository connection broken
6443  */
6444 static int
6445 process_pg_event(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst,
6446     char *pg_name)
6447 {
6448         int r;
6449         scf_property_t *prop;
6450         scf_value_t *val;
6451         char *fmri;
6452         boolean_t rebound = B_FALSE, rebind_inst = B_FALSE;
6453 
6454         if (scf_pg_get_name(pg, pg_name, max_scf_value_size) < 0) {
6455                 switch (scf_error()) {
6456                 case SCF_ERROR_CONNECTION_BROKEN:
6457                 default:
6458                         return (ECONNABORTED);
6459 
6460                 case SCF_ERROR_DELETED:
6461                         return (0);
6462 
6463                 case SCF_ERROR_NOT_SET:
6464                         bad_error("scf_pg_get_name", scf_error());
6465                 }
6466         }
6467 
6468         if (strcmp(pg_name, SCF_PG_GENERAL) == 0 ||
6469             strcmp(pg_name, SCF_PG_GENERAL_OVR) == 0) {
6470                 r = dgraph_update_general(pg);
6471                 switch (r) {
6472                 case 0:
6473                 case ENOTSUP:
6474                 case ECANCELED:
6475                         return (0);
6476 
6477                 case ECONNABORTED:
6478                         return (ECONNABORTED);
6479 
6480                 case -1:
6481                         /* Error should have been logged. */
6482                         return (0);
6483 
6484                 default:
6485                         bad_error("dgraph_update_general", r);
6486                 }
6487         } else if (strcmp(pg_name, SCF_PG_RESTARTER_ACTIONS) == 0) {
6488                 if (scf_pg_get_parent_instance(pg, inst) != 0) {
6489                         switch (scf_error()) {
6490                         case SCF_ERROR_CONNECTION_BROKEN:
6491                                 return (ECONNABORTED);
6492 
6493                         case SCF_ERROR_DELETED:
6494                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6495                                 /* Ignore commands on services. */
6496                                 return (0);
6497 
6498                         case SCF_ERROR_NOT_BOUND:
6499                         case SCF_ERROR_HANDLE_MISMATCH:
6500                         case SCF_ERROR_NOT_SET:
6501                         default:
6502                                 bad_error("scf_pg_get_parent_instance",
6503                                     scf_error());
6504                         }
6505                 }
6506 
6507                 return (process_actions(h, pg, inst));
6508         }
6509 
6510         if (strcmp(pg_name, SCF_PG_OPTIONS) != 0 &&
6511             strcmp(pg_name, SCF_PG_OPTIONS_OVR) != 0)
6512                 return (0);
6513 
6514         /*
6515          * We only care about the options[_ovr] property groups of our own
6516          * instance, so get the fmri and compare.  Plus, once we know it's
6517          * correct, if the repository connection is broken we know exactly what
6518          * property group we were operating on, and can look it up again.
6519          */
6520         if (scf_pg_get_parent_instance(pg, inst) != 0) {
6521                 switch (scf_error()) {
6522                 case SCF_ERROR_CONNECTION_BROKEN:
6523                         return (ECONNABORTED);
6524 
6525                 case SCF_ERROR_DELETED:
6526                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6527                         return (0);
6528 
6529                 case SCF_ERROR_HANDLE_MISMATCH:
6530                 case SCF_ERROR_NOT_BOUND:
6531                 case SCF_ERROR_NOT_SET:
6532                 default:
6533                         bad_error("scf_pg_get_parent_instance",
6534                             scf_error());
6535                 }
6536         }
6537 
6538         switch (r = libscf_instance_get_fmri(inst, &fmri)) {
6539         case 0:
6540                 break;
6541 
6542         case ECONNABORTED:
6543                 return (ECONNABORTED);
6544 
6545         case ECANCELED:
6546                 return (0);
6547 
6548         default:
6549                 bad_error("libscf_instance_get_fmri", r);
6550         }
6551 
6552         if (strcmp(fmri, SCF_SERVICE_STARTD) != 0) {
6553                 startd_free(fmri, max_scf_fmri_size);
6554                 return (0);
6555         }
6556 
6557         /*
6558          * update the information events flag
6559          */
6560         if (strcmp(pg_name, SCF_PG_OPTIONS) == 0)
6561                 info_events_all = libscf_get_info_events_all(pg);
6562 
6563         prop = safe_scf_property_create(h);
6564         val = safe_scf_value_create(h);
6565 
6566         if (strcmp(pg_name, SCF_PG_OPTIONS_OVR) == 0) {
6567                 /* See if we need to set the runlevel. */
6568                 /* CONSTCOND */
6569                 if (0) {
6570 rebind_pg:
6571                         libscf_handle_rebind(h);
6572                         rebound = B_TRUE;
6573 
6574                         r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6575                         switch (r) {
6576                         case 0:
6577                                 break;
6578 
6579                         case ECONNABORTED:
6580                                 goto rebind_pg;
6581 
6582                         case ENOENT:
6583                                 goto out;
6584 
6585                         case EINVAL:
6586                         case ENOTSUP:
6587                                 bad_error("libscf_lookup_instance", r);
6588                         }
6589 
6590                         if (scf_instance_get_pg(inst, pg_name, pg) != 0) {
6591                                 switch (scf_error()) {
6592                                 case SCF_ERROR_DELETED:
6593                                 case SCF_ERROR_NOT_FOUND:
6594                                         goto out;
6595 
6596                                 case SCF_ERROR_CONNECTION_BROKEN:
6597                                         goto rebind_pg;
6598 
6599                                 case SCF_ERROR_HANDLE_MISMATCH:
6600                                 case SCF_ERROR_NOT_BOUND:
6601                                 case SCF_ERROR_NOT_SET:
6602                                 case SCF_ERROR_INVALID_ARGUMENT:
6603                                 default:
6604                                         bad_error("scf_instance_get_pg",
6605                                             scf_error());
6606                                 }
6607                         }
6608                 }
6609 
6610                 if (scf_pg_get_property(pg, "runlevel", prop) == 0) {
6611                         r = dgraph_set_runlevel(pg, prop);
6612                         switch (r) {
6613                         case ECONNRESET:
6614                                 rebound = B_TRUE;
6615                                 rebind_inst = B_TRUE;
6616                                 /* FALLTHROUGH */
6617 
6618                         case 0:
6619                                 break;
6620 
6621                         case ECONNABORTED:
6622                                 goto rebind_pg;
6623 
6624                         case ECANCELED:
6625                                 goto out;
6626 
6627                         default:
6628                                 bad_error("dgraph_set_runlevel", r);
6629                         }
6630                 } else {
6631                         switch (scf_error()) {
6632                         case SCF_ERROR_CONNECTION_BROKEN:
6633                         default:
6634                                 goto rebind_pg;
6635 
6636                         case SCF_ERROR_DELETED:
6637                                 goto out;
6638 
6639                         case SCF_ERROR_NOT_FOUND:
6640                                 break;
6641 
6642                         case SCF_ERROR_INVALID_ARGUMENT:
6643                         case SCF_ERROR_HANDLE_MISMATCH:
6644                         case SCF_ERROR_NOT_BOUND:
6645                         case SCF_ERROR_NOT_SET:
6646                                 bad_error("scf_pg_get_property", scf_error());
6647                         }
6648                 }
6649         }
6650 
6651         if (rebind_inst) {
6652 lookup_inst:
6653                 r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6654                 switch (r) {
6655                 case 0:
6656                         break;
6657 
6658                 case ECONNABORTED:
6659                         libscf_handle_rebind(h);
6660                         rebound = B_TRUE;
6661                         goto lookup_inst;
6662 
6663                 case ENOENT:
6664                         goto out;
6665 
6666                 case EINVAL:
6667                 case ENOTSUP:
6668                         bad_error("libscf_lookup_instance", r);
6669                 }
6670         }
6671 
6672         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6673         switch (r) {
6674         case 0:
6675                 break;
6676 
6677         case ECONNABORTED:
6678                 libscf_handle_rebind(h);
6679                 rebound = B_TRUE;
6680                 goto lookup_inst;
6681 
6682         case EINVAL:
6683                 log_error(LOG_NOTICE,
6684                     "%s/%s property of %s is misconfigured.\n", pg_name,
6685                     SCF_PROPERTY_MILESTONE, SCF_SERVICE_STARTD);
6686                 /* FALLTHROUGH */
6687 
6688         case ECANCELED:
6689         case ENOENT:
6690                 (void) strcpy(fmri, "all");
6691                 break;
6692 
6693         default:
6694                 bad_error("libscf_get_milestone", r);
6695         }
6696 
6697         r = dgraph_set_milestone(fmri, h, B_FALSE);
6698         switch (r) {
6699         case 0:
6700         case ECONNRESET:
6701         case EALREADY:
6702                 break;
6703 
6704         case EINVAL:
6705                 log_error(LOG_WARNING, "Milestone %s is invalid.\n", fmri);
6706                 break;
6707 
6708         case ENOENT:
6709                 log_error(LOG_WARNING, "Milestone %s does not exist.\n", fmri);
6710                 break;
6711 
6712         default:
6713                 bad_error("dgraph_set_milestone", r);
6714         }
6715 
6716 out:
6717         startd_free(fmri, max_scf_fmri_size);
6718         scf_value_destroy(val);
6719         scf_property_destroy(prop);
6720 
6721         return (rebound ? ECONNRESET : 0);
6722 }
6723 
6724 /*
6725  * process_delete() deletes an instance from the dgraph if 'fmri' is an
6726  * instance fmri or if 'fmri' matches the 'general' property group of an
6727  * instance (or the 'general/enabled' property).
6728  *
6729  * 'fmri' may be overwritten and cannot be trusted on return by the caller.
6730  */
6731 static void
6732 process_delete(char *fmri, scf_handle_t *h)
6733 {
6734         char *lfmri, *end_inst_fmri;
6735         const char *inst_name = NULL;
6736         const char *pg_name = NULL;
6737         const char *prop_name = NULL;
6738 
6739         lfmri = safe_strdup(fmri);
6740 
6741         /* Determine if the FMRI is a property group or instance */
6742         if (scf_parse_svc_fmri(lfmri, NULL, NULL, &inst_name, &pg_name,
6743             &prop_name) != SCF_SUCCESS) {
6744                 log_error(LOG_WARNING,
6745                     "Received invalid FMRI \"%s\" from repository server.\n",
6746                     fmri);
6747         } else if (inst_name != NULL && pg_name == NULL) {
6748                 (void) dgraph_remove_instance(fmri, h);
6749         } else if (inst_name != NULL && pg_name != NULL) {
6750                 /*
6751                  * If we're deleting the 'general' property group or
6752                  * 'general/enabled' property then the whole instance
6753                  * must be removed from the dgraph.
6754                  */
6755                 if (strcmp(pg_name, SCF_PG_GENERAL) != 0) {
6756                         free(lfmri);
6757                         return;
6758                 }
6759 
6760                 if (prop_name != NULL &&
6761                     strcmp(prop_name, SCF_PROPERTY_ENABLED) != 0) {
6762                         free(lfmri);
6763                         return;
6764                 }
6765 
6766                 /*
6767                  * Because the instance has already been deleted from the
6768                  * repository, we cannot use any scf_ functions to retrieve
6769                  * the instance FMRI however we can easily reconstruct it
6770                  * manually.
6771                  */
6772                 end_inst_fmri = strstr(fmri, SCF_FMRI_PROPERTYGRP_PREFIX);
6773                 if (end_inst_fmri == NULL)
6774                         bad_error("process_delete", 0);
6775 
6776                 end_inst_fmri[0] = '\0';
6777 
6778                 (void) dgraph_remove_instance(fmri, h);
6779         }
6780 
6781         free(lfmri);
6782 }
6783 
6784 /*ARGSUSED*/
6785 void *
6786 repository_event_thread(void *unused)
6787 {
6788         scf_handle_t *h;
6789         scf_propertygroup_t *pg;
6790         scf_instance_t *inst;
6791         char *fmri = startd_alloc(max_scf_fmri_size);
6792         char *pg_name = startd_alloc(max_scf_value_size);
6793         int r;
6794 
6795         h = libscf_handle_create_bound_loop();
6796 
6797         pg = safe_scf_pg_create(h);
6798         inst = safe_scf_instance_create(h);
6799 
6800 retry:
6801         if (_scf_notify_add_pgtype(h, SCF_GROUP_FRAMEWORK) != SCF_SUCCESS) {
6802                 if (scf_error() == SCF_ERROR_CONNECTION_BROKEN) {
6803                         libscf_handle_rebind(h);
6804                 } else {
6805                         log_error(LOG_WARNING,
6806                             "Couldn't set up repository notification "
6807                             "for property group type %s: %s\n",
6808                             SCF_GROUP_FRAMEWORK, scf_strerror(scf_error()));
6809 
6810                         (void) sleep(1);
6811                 }
6812 
6813                 goto retry;
6814         }
6815 
6816         /*CONSTCOND*/
6817         while (1) {
6818                 ssize_t res;
6819 
6820                 /* Note: fmri is only set on delete events. */
6821                 res = _scf_notify_wait(pg, fmri, max_scf_fmri_size);
6822                 if (res < 0) {
6823                         libscf_handle_rebind(h);
6824                         goto retry;
6825                 } else if (res == 0) {
6826                         /*
6827                          * property group modified.  inst and pg_name are
6828                          * pre-allocated scratch space.
6829                          */
6830                         if (scf_pg_update(pg) < 0) {
6831                                 switch (scf_error()) {
6832                                 case SCF_ERROR_DELETED:
6833                                         continue;
6834 
6835                                 case SCF_ERROR_CONNECTION_BROKEN:
6836                                         log_error(LOG_WARNING,
6837                                             "Lost repository event due to "
6838                                             "disconnection.\n");
6839                                         libscf_handle_rebind(h);
6840                                         goto retry;
6841 
6842                                 case SCF_ERROR_NOT_BOUND:
6843                                 case SCF_ERROR_NOT_SET:
6844                                 default:
6845                                         bad_error("scf_pg_update", scf_error());
6846                                 }
6847                         }
6848 
6849                         r = process_pg_event(h, pg, inst, pg_name);
6850                         switch (r) {
6851                         case 0:
6852                                 break;
6853 
6854                         case ECONNABORTED:
6855                                 log_error(LOG_WARNING, "Lost repository event "
6856                                     "due to disconnection.\n");
6857                                 libscf_handle_rebind(h);
6858                                 /* FALLTHROUGH */
6859 
6860                         case ECONNRESET:
6861                                 goto retry;
6862 
6863                         default:
6864                                 bad_error("process_pg_event", r);
6865                         }
6866                 } else {
6867                         /*
6868                          * Service, instance, or pg deleted.
6869                          * Don't trust fmri on return.
6870                          */
6871                         process_delete(fmri, h);
6872                 }
6873         }
6874 
6875         /*NOTREACHED*/
6876         return (NULL);
6877 }
6878 
6879 void
6880 graph_engine_start()
6881 {
6882         int err;
6883 
6884         (void) startd_thread_create(graph_thread, NULL);
6885 
6886         MUTEX_LOCK(&dgraph_lock);
6887         while (!initial_milestone_set) {
6888                 err = pthread_cond_wait(&initial_milestone_cv, &dgraph_lock);
6889                 assert(err == 0);
6890         }
6891         MUTEX_UNLOCK(&dgraph_lock);
6892 
6893         (void) startd_thread_create(repository_event_thread, NULL);
6894         (void) startd_thread_create(graph_event_thread, NULL);
6895 }