1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
  25  * Copyright 2016 RackTop Systems.
  26  */
  27 
  28 /*
  29  * graph.c - master restarter graph engine
  30  *
  31  *   The graph engine keeps a dependency graph of all service instances on the
  32  *   system, as recorded in the repository.  It decides when services should
  33  *   be brought up or down based on service states and dependencies and sends
  34  *   commands to restarters to effect any changes.  It also executes
  35  *   administrator commands sent by svcadm via the repository.
  36  *
  37  *   The graph is stored in uu_list_t *dgraph and its vertices are
  38  *   graph_vertex_t's, each of which has a name and an integer id unique to
  39  *   its name (see dict.c).  A vertex's type attribute designates the type
  40  *   of object it represents: GVT_INST for service instances, GVT_SVC for
  41  *   service objects (since service instances may depend on another service,
  42  *   rather than service instance), GVT_FILE for files (which services may
  43  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
  44  *   vertices are necessary because dependency lists may have particular
  45  *   grouping types (require any, require all, optional, or exclude) and
  46  *   event-propagation characteristics.
  47  *
  48  *   The initial graph is built by libscf_populate_graph() invoking
  49  *   dgraph_add_instance() for each instance in the repository.  The function
  50  *   adds a GVT_SVC vertex for the service if one does not already exist, adds
  51  *   a GVT_INST vertex named by the FMRI of the instance, and sets up the edges.
  52  *   The resulting web of vertices & edges associated with an instance's vertex
  53  *   includes
  54  *
  55  *     - an edge from the GVT_SVC vertex for the instance's service
  56  *
  57  *     - an edge to the GVT_INST vertex of the instance's resarter, if its
  58  *       restarter is not svc.startd
  59  *
  60  *     - edges from other GVT_INST vertices if the instance is a restarter
  61  *
  62  *     - for each dependency property group in the instance's "running"
  63  *       snapshot, an edge to a GVT_GROUP vertex named by the FMRI of the
  64  *       instance and the name of the property group
  65  *
  66  *     - for each value of the "entities" property in each dependency property
  67  *       group, an edge from the corresponding GVT_GROUP vertex to a
  68  *       GVT_INST, GVT_SVC, or GVT_FILE vertex
  69  *
  70  *     - edges from GVT_GROUP vertices for each dependent instance
  71  *
  72  *   After the edges are set up the vertex's GV_CONFIGURED flag is set.  If
  73  *   there are problems, or if a service is mentioned in a dependency but does
  74  *   not exist in the repository, the GV_CONFIGURED flag will be clear.
  75  *
  76  *   The graph and all of its vertices are protected by the dgraph_lock mutex.
  77  *   See restarter.c for more information.
  78  *
  79  *   The properties of an instance fall into two classes: immediate and
  80  *   snapshotted.  Immediate properties should have an immediate effect when
  81  *   changed.  Snapshotted properties should be read from a snapshot, so they
  82  *   only change when the snapshot changes.  The immediate properties used by
  83  *   the graph engine are general/enabled, general/restarter, and the properties
  84  *   in the restarter_actions property group.  Since they are immediate, they
  85  *   are not read out of a snapshot.  The snapshotted properties used by the
  86  *   graph engine are those in the property groups with type "dependency" and
  87  *   are read out of the "running" snapshot.  The "running" snapshot is created
  88  *   by the the graph engine as soon as possible, and it is updated, along with
  89  *   in-core copies of the data (dependency information for the graph engine) on
  90  *   receipt of the refresh command from svcadm.  In addition, the graph engine
  91  *   updates the "start" snapshot from the "running" snapshot whenever a service
  92  *   comes online.
  93  *
  94  *   When a DISABLE event is requested by the administrator, svc.startd shutdown
  95  *   the dependents first before shutting down the requested service.
  96  *   In graph_enable_by_vertex, we create a subtree that contains the dependent
  97  *   vertices by marking those vertices with the GV_TOOFFLINE flag. And we mark
  98  *   the vertex to disable with the GV_TODISABLE flag. Once the tree is created,
  99  *   we send the _ADMIN_DISABLE event to the leaves. The leaves will then
 100  *   transition from STATE_ONLINE/STATE_DEGRADED to STATE_OFFLINE/STATE_MAINT.
 101  *   In gt_enter_offline and gt_enter_maint if the vertex was in a subtree then
 102  *   we clear the GV_TOOFFLINE flag and walk the dependencies to offline the new
 103  *   exposed leaves. We do the same until we reach the last leaf (the one with
 104  *   the GV_TODISABLE flag). If the vertex to disable is also part of a larger
 105  *   subtree (eg. multiple DISABLE events on vertices in the same subtree) then
 106  *   once the first vertex is disabled (GV_TODISABLE flag is removed), we
 107  *   continue to propagate the offline event to the vertex's dependencies.
 108  *
 109  *
 110  * SMF state transition notifications
 111  *
 112  *   When an instance of a service managed by SMF changes state, svc.startd may
 113  *   publish a GPEC sysevent. All transitions to or from maintenance, a
 114  *   transition cause by a hardware error will generate an event.
 115  *   Other transitions will generate an event if there exist notification
 116  *   parameter for that transition. Notification parameters are stored in the
 117  *   SMF repository for the service/instance they refer to. System-wide
 118  *   notification parameters are stored in the global instance.
 119  *   svc.startd can be told to send events for all SMF state transitions despite
 120  *   of notification parameters by setting options/info_events_all to true in
 121  *   restarter:default
 122  *
 123  *   The set of transitions that generate events is cached in the
 124  *   dgraph_vertex_t gv_stn_tset for service/instance and in the global
 125  *   stn_global for the system-wide set. They are re-read when instances are
 126  *   refreshed.
 127  *
 128  *   The GPEC events published by svc.startd are consumed by fmd(1M). After
 129  *   processing these events, fmd(1M) publishes the processed events to
 130  *   notification agents. The notification agents read the notification
 131  *   parameters from the SMF repository through libscf(3LIB) interfaces and send
 132  *   the notification, or not, based on those parameters.
 133  *
 134  *   Subscription and publishing to the GPEC channels is done with the
 135  *   libfmevent(3LIB) wrappers fmev_[r]publish_*() and
 136  *   fmev_shdl_(un)subscribe().
 137  *
 138  */
 139 
 140 #include <sys/uadmin.h>
 141 #include <sys/wait.h>
 142 
 143 #include <assert.h>
 144 #include <errno.h>
 145 #include <fcntl.h>
 146 #include <fm/libfmevent.h>
 147 #include <libscf.h>
 148 #include <libscf_priv.h>
 149 #include <librestart.h>
 150 #include <libuutil.h>
 151 #include <locale.h>
 152 #include <poll.h>
 153 #include <pthread.h>
 154 #include <signal.h>
 155 #include <stddef.h>
 156 #include <stdio.h>
 157 #include <stdlib.h>
 158 #include <string.h>
 159 #include <strings.h>
 160 #include <sys/statvfs.h>
 161 #include <sys/uadmin.h>
 162 #include <zone.h>
 163 #if defined(__i386)
 164 #include <libgrubmgmt.h>
 165 #endif  /* __i386 */
 166 
 167 #include "startd.h"
 168 #include "protocol.h"
 169 
 170 
 171 #define MILESTONE_NONE  ((graph_vertex_t *)1)
 172 
 173 #define CONSOLE_LOGIN_FMRI      "svc:/system/console-login:default"
 174 #define FS_MINIMAL_FMRI         "svc:/system/filesystem/minimal:default"
 175 
 176 #define VERTEX_REMOVED  0       /* vertex has been freed  */
 177 #define VERTEX_INUSE    1       /* vertex is still in use */
 178 
 179 #define IS_ENABLED(v) ((v)->gv_flags & (GV_ENABLED | GV_ENBLD_NOOVR))
 180 
 181 /*
 182  * stn_global holds the tset for the system wide notification parameters.
 183  * It is updated on refresh of svc:/system/svc/global:default
 184  *
 185  * There are two assumptions that relax the need for a mutex:
 186  *     1. 32-bit value assignments are atomic
 187  *     2. Its value is consumed only in one point at
 188  *     dgraph_state_transition_notify(). There are no test and set races.
 189  *
 190  *     If either assumption is broken, we'll need a mutex to synchronize
 191  *     access to stn_global
 192  */
 193 int32_t stn_global;
 194 /*
 195  * info_events_all holds a flag to override notification parameters and send
 196  * Information events for all state transitions.
 197  * same about the need of a mutex here.
 198  */
 199 int info_events_all;
 200 
 201 /*
 202  * Services in these states are not considered 'down' by the
 203  * milestone/shutdown code.
 204  */
 205 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
 206         (state) == RESTARTER_STATE_DEGRADED || \
 207         (state) == RESTARTER_STATE_OFFLINE)
 208 
 209 #define is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
 210         ((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \
 211         (v->gv_restart < RERR_RESTART)))
 212 
 213 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
 214 static uu_list_t *dgraph;
 215 static pthread_mutex_t dgraph_lock;
 216 
 217 /*
 218  * milestone indicates the current subgraph.  When NULL, it is the entire
 219  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
 220  * services on which the target vertex depends.
 221  */
 222 static graph_vertex_t *milestone = NULL;
 223 static boolean_t initial_milestone_set = B_FALSE;
 224 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
 225 
 226 /* protected by dgraph_lock */
 227 static boolean_t sulogin_thread_running = B_FALSE;
 228 static boolean_t sulogin_running = B_FALSE;
 229 static boolean_t console_login_ready = B_FALSE;
 230 
 231 /* Number of services to come down to complete milestone transition. */
 232 static uint_t non_subgraph_svcs;
 233 
 234 /*
 235  * These variables indicate what should be done when we reach the milestone
 236  * target milestone, i.e., when non_subgraph_svcs == 0.  They are acted upon in
 237  * dgraph_set_instance_state().
 238  */
 239 static int halting = -1;
 240 static boolean_t go_single_user_mode = B_FALSE;
 241 static boolean_t go_to_level1 = B_FALSE;
 242 
 243 /*
 244  * Tracks when we started halting.
 245  */
 246 static time_t halting_time = 0;
 247 
 248 /*
 249  * This tracks the legacy runlevel to ensure we signal init and manage
 250  * utmpx entries correctly.
 251  */
 252 static char current_runlevel = '\0';
 253 
 254 /* Number of single user threads currently running */
 255 static pthread_mutex_t single_user_thread_lock;
 256 static int single_user_thread_count = 0;
 257 
 258 /* Statistics for dependency cycle-checking */
 259 static u_longlong_t dep_inserts = 0;
 260 static u_longlong_t dep_cycle_ns = 0;
 261 static u_longlong_t dep_insert_ns = 0;
 262 
 263 
 264 static const char * const emsg_invalid_restarter =
 265         "Transitioning %s to maintenance, restarter FMRI %s is invalid "
 266         "(see 'svcs -xv' for details).\n";
 267 static const char * const console_login_fmri = CONSOLE_LOGIN_FMRI;
 268 static const char * const single_user_fmri = SCF_MILESTONE_SINGLE_USER;
 269 static const char * const multi_user_fmri = SCF_MILESTONE_MULTI_USER;
 270 static const char * const multi_user_svr_fmri = SCF_MILESTONE_MULTI_USER_SERVER;
 271 
 272 
 273 /*
 274  * These services define the system being "up".  If none of them can come
 275  * online, then we will run sulogin on the console.  Note that the install ones
 276  * are for the miniroot and when installing CDs after the first.  can_come_up()
 277  * does the decision making, and an sulogin_thread() runs sulogin, which can be
 278  * started by dgraph_set_instance_state() or single_user_thread().
 279  *
 280  * NOTE: can_come_up() relies on SCF_MILESTONE_SINGLE_USER being the first
 281  * entry, which is only used when booting_to_single_user (boot -s) is set.
 282  * This is because when doing a "boot -s", sulogin is started from specials.c
 283  * after milestone/single-user comes online, for backwards compatibility.
 284  * In this case, SCF_MILESTONE_SINGLE_USER needs to be part of up_svcs
 285  * to ensure sulogin will be spawned if milestone/single-user cannot be reached.
 286  */
 287 static const char * const up_svcs[] = {
 288         SCF_MILESTONE_SINGLE_USER,
 289         CONSOLE_LOGIN_FMRI,
 290         "svc:/system/install-setup:default",
 291         "svc:/system/install:default",
 292         NULL
 293 };
 294 
 295 /* This array must have an element for each non-NULL element of up_svcs[]. */
 296 static graph_vertex_t *up_svcs_p[] = { NULL, NULL, NULL, NULL };
 297 
 298 /* These are for seed repository magic.  See can_come_up(). */
 299 static const char * const manifest_import = SCF_INSTANCE_MI;
 300 static graph_vertex_t *manifest_import_p = NULL;
 301 
 302 
 303 static char target_milestone_as_runlevel(void);
 304 static void graph_runlevel_changed(char rl, int online);
 305 static int dgraph_set_milestone(const char *, scf_handle_t *, boolean_t);
 306 static boolean_t should_be_in_subgraph(graph_vertex_t *v);
 307 static int mark_subtree(graph_edge_t *, void *);
 308 static boolean_t insubtree_dependents_down(graph_vertex_t *);
 309 
 310 /*
 311  * graph_vertex_compare()
 312  *      This function can compare either int *id or * graph_vertex_t *gv
 313  *      values, as the vertex id is always the first element of a
 314  *      graph_vertex structure.
 315  */
 316 /* ARGSUSED */
 317 static int
 318 graph_vertex_compare(const void *lc_arg, const void *rc_arg, void *private)
 319 {
 320         int lc_id = ((const graph_vertex_t *)lc_arg)->gv_id;
 321         int rc_id = *(int *)rc_arg;
 322 
 323         if (lc_id > rc_id)
 324                 return (1);
 325         if (lc_id < rc_id)
 326                 return (-1);
 327         return (0);
 328 }
 329 
 330 void
 331 graph_init()
 332 {
 333         graph_edge_pool = startd_list_pool_create("graph_edges",
 334             sizeof (graph_edge_t), offsetof(graph_edge_t, ge_link), NULL,
 335             UU_LIST_POOL_DEBUG);
 336         assert(graph_edge_pool != NULL);
 337 
 338         graph_vertex_pool = startd_list_pool_create("graph_vertices",
 339             sizeof (graph_vertex_t), offsetof(graph_vertex_t, gv_link),
 340             graph_vertex_compare, UU_LIST_POOL_DEBUG);
 341         assert(graph_vertex_pool != NULL);
 342 
 343         (void) pthread_mutex_init(&dgraph_lock, &mutex_attrs);
 344         (void) pthread_mutex_init(&single_user_thread_lock, &mutex_attrs);
 345         dgraph = startd_list_create(graph_vertex_pool, NULL, UU_LIST_SORTED);
 346         assert(dgraph != NULL);
 347 
 348         if (!st->st_initial)
 349                 current_runlevel = utmpx_get_runlevel();
 350 
 351         log_framework(LOG_DEBUG, "Initialized graph\n");
 352 }
 353 
 354 static graph_vertex_t *
 355 vertex_get_by_name(const char *name)
 356 {
 357         int id;
 358 
 359         assert(MUTEX_HELD(&dgraph_lock));
 360 
 361         id = dict_lookup_byname(name);
 362         if (id == -1)
 363                 return (NULL);
 364 
 365         return (uu_list_find(dgraph, &id, NULL, NULL));
 366 }
 367 
 368 static graph_vertex_t *
 369 vertex_get_by_id(int id)
 370 {
 371         assert(MUTEX_HELD(&dgraph_lock));
 372 
 373         if (id == -1)
 374                 return (NULL);
 375 
 376         return (uu_list_find(dgraph, &id, NULL, NULL));
 377 }
 378 
 379 /*
 380  * Creates a new vertex with the given name, adds it to the graph, and returns
 381  * a pointer to it.  The graph lock must be held by this thread on entry.
 382  */
 383 static graph_vertex_t *
 384 graph_add_vertex(const char *name)
 385 {
 386         int id;
 387         graph_vertex_t *v;
 388         void *p;
 389         uu_list_index_t idx;
 390 
 391         assert(MUTEX_HELD(&dgraph_lock));
 392 
 393         id = dict_insert(name);
 394 
 395         v = startd_zalloc(sizeof (*v));
 396 
 397         v->gv_id = id;
 398 
 399         v->gv_name = startd_alloc(strlen(name) + 1);
 400         (void) strcpy(v->gv_name, name);
 401 
 402         v->gv_dependencies = startd_list_create(graph_edge_pool, v, 0);
 403         v->gv_dependents = startd_list_create(graph_edge_pool, v, 0);
 404 
 405         p = uu_list_find(dgraph, &id, NULL, &idx);
 406         assert(p == NULL);
 407 
 408         uu_list_node_init(v, &v->gv_link, graph_vertex_pool);
 409         uu_list_insert(dgraph, v, idx);
 410 
 411         return (v);
 412 }
 413 
 414 /*
 415  * Removes v from the graph and frees it.  The graph should be locked by this
 416  * thread, and v should have no edges associated with it.
 417  */
 418 static void
 419 graph_remove_vertex(graph_vertex_t *v)
 420 {
 421         assert(MUTEX_HELD(&dgraph_lock));
 422 
 423         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 424         assert(uu_list_numnodes(v->gv_dependents) == 0);
 425         assert(v->gv_refs == 0);
 426 
 427         startd_free(v->gv_name, strlen(v->gv_name) + 1);
 428         uu_list_destroy(v->gv_dependencies);
 429         uu_list_destroy(v->gv_dependents);
 430         uu_list_remove(dgraph, v);
 431 
 432         startd_free(v, sizeof (graph_vertex_t));
 433 }
 434 
 435 static void
 436 graph_add_edge(graph_vertex_t *fv, graph_vertex_t *tv)
 437 {
 438         graph_edge_t *e, *re;
 439         int r;
 440 
 441         assert(MUTEX_HELD(&dgraph_lock));
 442 
 443         e = startd_alloc(sizeof (graph_edge_t));
 444         re = startd_alloc(sizeof (graph_edge_t));
 445 
 446         e->ge_parent = fv;
 447         e->ge_vertex = tv;
 448 
 449         re->ge_parent = tv;
 450         re->ge_vertex = fv;
 451 
 452         uu_list_node_init(e, &e->ge_link, graph_edge_pool);
 453         r = uu_list_insert_before(fv->gv_dependencies, NULL, e);
 454         assert(r == 0);
 455 
 456         uu_list_node_init(re, &re->ge_link, graph_edge_pool);
 457         r = uu_list_insert_before(tv->gv_dependents, NULL, re);
 458         assert(r == 0);
 459 }
 460 
 461 static void
 462 graph_remove_edge(graph_vertex_t *v, graph_vertex_t *dv)
 463 {
 464         graph_edge_t *e;
 465 
 466         for (e = uu_list_first(v->gv_dependencies);
 467             e != NULL;
 468             e = uu_list_next(v->gv_dependencies, e)) {
 469                 if (e->ge_vertex == dv) {
 470                         uu_list_remove(v->gv_dependencies, e);
 471                         startd_free(e, sizeof (graph_edge_t));
 472                         break;
 473                 }
 474         }
 475 
 476         for (e = uu_list_first(dv->gv_dependents);
 477             e != NULL;
 478             e = uu_list_next(dv->gv_dependents, e)) {
 479                 if (e->ge_vertex == v) {
 480                         uu_list_remove(dv->gv_dependents, e);
 481                         startd_free(e, sizeof (graph_edge_t));
 482                         break;
 483                 }
 484         }
 485 }
 486 
 487 static void
 488 remove_inst_vertex(graph_vertex_t *v)
 489 {
 490         graph_edge_t *e;
 491         graph_vertex_t *sv;
 492         int i;
 493 
 494         assert(MUTEX_HELD(&dgraph_lock));
 495         assert(uu_list_numnodes(v->gv_dependents) == 1);
 496         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 497         assert(v->gv_refs == 0);
 498         assert((v->gv_flags & GV_CONFIGURED) == 0);
 499 
 500         e = uu_list_first(v->gv_dependents);
 501         sv = e->ge_vertex;
 502         graph_remove_edge(sv, v);
 503 
 504         for (i = 0; up_svcs[i] != NULL; ++i) {
 505                 if (up_svcs_p[i] == v)
 506                         up_svcs_p[i] = NULL;
 507         }
 508 
 509         if (manifest_import_p == v)
 510                 manifest_import_p = NULL;
 511 
 512         graph_remove_vertex(v);
 513 
 514         if (uu_list_numnodes(sv->gv_dependencies) == 0 &&
 515             uu_list_numnodes(sv->gv_dependents) == 0 &&
 516             sv->gv_refs == 0)
 517                 graph_remove_vertex(sv);
 518 }
 519 
 520 static void
 521 graph_walk_dependents(graph_vertex_t *v, void (*func)(graph_vertex_t *, void *),
 522     void *arg)
 523 {
 524         graph_edge_t *e;
 525 
 526         for (e = uu_list_first(v->gv_dependents);
 527             e != NULL;
 528             e = uu_list_next(v->gv_dependents, e))
 529                 func(e->ge_vertex, arg);
 530 }
 531 
 532 static void
 533 graph_walk_dependencies(graph_vertex_t *v, void (*func)(graph_vertex_t *,
 534         void *), void *arg)
 535 {
 536         graph_edge_t *e;
 537 
 538         assert(MUTEX_HELD(&dgraph_lock));
 539 
 540         for (e = uu_list_first(v->gv_dependencies);
 541             e != NULL;
 542             e = uu_list_next(v->gv_dependencies, e)) {
 543 
 544                 func(e->ge_vertex, arg);
 545         }
 546 }
 547 
 548 /*
 549  * Generic graph walking function.
 550  *
 551  * Given a vertex, this function will walk either dependencies
 552  * (WALK_DEPENDENCIES) or dependents (WALK_DEPENDENTS) of a vertex recursively
 553  * for the entire graph.  It will avoid cycles and never visit the same vertex
 554  * twice.
 555  *
 556  * We avoid traversing exclusion dependencies, because they are allowed to
 557  * create cycles in the graph.  When propagating satisfiability, there is no
 558  * need to walk exclusion dependencies because exclude_all_satisfied() doesn't
 559  * test for satisfiability.
 560  *
 561  * The walker takes two callbacks.  The first is called before examining the
 562  * dependents of each vertex.  The second is called on each vertex after
 563  * examining its dependents.  This allows is_path_to() to construct a path only
 564  * after the target vertex has been found.
 565  */
 566 typedef enum {
 567         WALK_DEPENDENTS,
 568         WALK_DEPENDENCIES
 569 } graph_walk_dir_t;
 570 
 571 typedef int (*graph_walk_cb_t)(graph_vertex_t *, void *);
 572 
 573 typedef struct graph_walk_info {
 574         graph_walk_dir_t        gi_dir;
 575         uchar_t                 *gi_visited;    /* vertex bitmap */
 576         int                     (*gi_pre)(graph_vertex_t *, void *);
 577         void                    (*gi_post)(graph_vertex_t *, void *);
 578         void                    *gi_arg;        /* callback arg */
 579         int                     gi_ret;         /* return value */
 580 } graph_walk_info_t;
 581 
 582 static int
 583 graph_walk_recurse(graph_edge_t *e, graph_walk_info_t *gip)
 584 {
 585         uu_list_t *list;
 586         int r;
 587         graph_vertex_t *v = e->ge_vertex;
 588         int i;
 589         uint_t b;
 590 
 591         i = v->gv_id / 8;
 592         b = 1 << (v->gv_id % 8);
 593 
 594         /*
 595          * Check to see if we've visited this vertex already.
 596          */
 597         if (gip->gi_visited[i] & b)
 598                 return (UU_WALK_NEXT);
 599 
 600         gip->gi_visited[i] |= b;
 601 
 602         /*
 603          * Don't follow exclusions.
 604          */
 605         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
 606                 return (UU_WALK_NEXT);
 607 
 608         /*
 609          * Call pre-visit callback.  If this doesn't terminate the walk,
 610          * continue search.
 611          */
 612         if ((gip->gi_ret = gip->gi_pre(v, gip->gi_arg)) == UU_WALK_NEXT) {
 613                 /*
 614                  * Recurse using appropriate list.
 615                  */
 616                 if (gip->gi_dir == WALK_DEPENDENTS)
 617                         list = v->gv_dependents;
 618                 else
 619                         list = v->gv_dependencies;
 620 
 621                 r = uu_list_walk(list, (uu_walk_fn_t *)graph_walk_recurse,
 622                     gip, 0);
 623                 assert(r == 0);
 624         }
 625 
 626         /*
 627          * Callbacks must return either UU_WALK_NEXT or UU_WALK_DONE.
 628          */
 629         assert(gip->gi_ret == UU_WALK_NEXT || gip->gi_ret == UU_WALK_DONE);
 630 
 631         /*
 632          * If given a post-callback, call the function for every vertex.
 633          */
 634         if (gip->gi_post != NULL)
 635                 (void) gip->gi_post(v, gip->gi_arg);
 636 
 637         /*
 638          * Preserve the callback's return value.  If the callback returns
 639          * UU_WALK_DONE, then we propagate that to the caller in order to
 640          * terminate the walk.
 641          */
 642         return (gip->gi_ret);
 643 }
 644 
 645 static void
 646 graph_walk(graph_vertex_t *v, graph_walk_dir_t dir,
 647     int (*pre)(graph_vertex_t *, void *),
 648     void (*post)(graph_vertex_t *, void *), void *arg)
 649 {
 650         graph_walk_info_t gi;
 651         graph_edge_t fake;
 652         size_t sz = dictionary->dict_new_id / 8 + 1;
 653 
 654         gi.gi_visited = startd_zalloc(sz);
 655         gi.gi_pre = pre;
 656         gi.gi_post = post;
 657         gi.gi_arg = arg;
 658         gi.gi_dir = dir;
 659         gi.gi_ret = 0;
 660 
 661         /*
 662          * Fake up an edge for the first iteration
 663          */
 664         fake.ge_vertex = v;
 665         (void) graph_walk_recurse(&fake, &gi);
 666 
 667         startd_free(gi.gi_visited, sz);
 668 }
 669 
 670 typedef struct child_search {
 671         int     id;             /* id of vertex to look for */
 672         uint_t  depth;          /* recursion depth */
 673         /*
 674          * While the vertex is not found, path is NULL.  After the search, if
 675          * the vertex was found then path should point to a -1-terminated
 676          * array of vertex id's which constitute the path to the vertex.
 677          */
 678         int     *path;
 679 } child_search_t;
 680 
 681 static int
 682 child_pre(graph_vertex_t *v, void *arg)
 683 {
 684         child_search_t *cs = arg;
 685 
 686         cs->depth++;
 687 
 688         if (v->gv_id == cs->id) {
 689                 cs->path = startd_alloc((cs->depth + 1) * sizeof (int));
 690                 cs->path[cs->depth] = -1;
 691                 return (UU_WALK_DONE);
 692         }
 693 
 694         return (UU_WALK_NEXT);
 695 }
 696 
 697 static void
 698 child_post(graph_vertex_t *v, void *arg)
 699 {
 700         child_search_t *cs = arg;
 701 
 702         cs->depth--;
 703 
 704         if (cs->path != NULL)
 705                 cs->path[cs->depth] = v->gv_id;
 706 }
 707 
 708 /*
 709  * Look for a path from from to to.  If one exists, returns a pointer to
 710  * a NULL-terminated array of pointers to the vertices along the path.  If
 711  * there is no path, returns NULL.
 712  */
 713 static int *
 714 is_path_to(graph_vertex_t *from, graph_vertex_t *to)
 715 {
 716         child_search_t cs;
 717 
 718         cs.id = to->gv_id;
 719         cs.depth = 0;
 720         cs.path = NULL;
 721 
 722         graph_walk(from, WALK_DEPENDENCIES, child_pre, child_post, &cs);
 723 
 724         return (cs.path);
 725 }
 726 
 727 /*
 728  * Given an array of int's as returned by is_path_to, allocates a string of
 729  * their names joined by newlines.  Returns the size of the allocated buffer
 730  * in *sz and frees path.
 731  */
 732 static void
 733 path_to_str(int *path, char **cpp, size_t *sz)
 734 {
 735         int i;
 736         graph_vertex_t *v;
 737         size_t allocd, new_allocd;
 738         char *new, *name;
 739 
 740         assert(MUTEX_HELD(&dgraph_lock));
 741         assert(path[0] != -1);
 742 
 743         allocd = 1;
 744         *cpp = startd_alloc(1);
 745         (*cpp)[0] = '\0';
 746 
 747         for (i = 0; path[i] != -1; ++i) {
 748                 name = NULL;
 749 
 750                 v = vertex_get_by_id(path[i]);
 751 
 752                 if (v == NULL)
 753                         name = "<deleted>";
 754                 else if (v->gv_type == GVT_INST || v->gv_type == GVT_SVC)
 755                         name = v->gv_name;
 756 
 757                 if (name != NULL) {
 758                         new_allocd = allocd + strlen(name) + 1;
 759                         new = startd_alloc(new_allocd);
 760                         (void) strcpy(new, *cpp);
 761                         (void) strcat(new, name);
 762                         (void) strcat(new, "\n");
 763 
 764                         startd_free(*cpp, allocd);
 765 
 766                         *cpp = new;
 767                         allocd = new_allocd;
 768                 }
 769         }
 770 
 771         startd_free(path, sizeof (int) * (i + 1));
 772 
 773         *sz = allocd;
 774 }
 775 
 776 
 777 /*
 778  * This function along with run_sulogin() implements an exclusion relationship
 779  * between system/console-login and sulogin.  run_sulogin() will fail if
 780  * system/console-login is online, and the graph engine should call
 781  * graph_clogin_start() to bring system/console-login online, which defers the
 782  * start if sulogin is running.
 783  */
 784 static void
 785 graph_clogin_start(graph_vertex_t *v)
 786 {
 787         assert(MUTEX_HELD(&dgraph_lock));
 788 
 789         if (sulogin_running)
 790                 console_login_ready = B_TRUE;
 791         else
 792                 vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 793 }
 794 
 795 static void
 796 graph_su_start(graph_vertex_t *v)
 797 {
 798         /*
 799          * /etc/inittab used to have the initial /sbin/rcS as a 'sysinit'
 800          * entry with a runlevel of 'S', before jumping to the final
 801          * target runlevel (as set in initdefault).  We mimic that legacy
 802          * behavior here.
 803          */
 804         utmpx_set_runlevel('S', '0', B_FALSE);
 805         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 806 }
 807 
 808 static void
 809 graph_post_su_online(void)
 810 {
 811         graph_runlevel_changed('S', 1);
 812 }
 813 
 814 static void
 815 graph_post_su_disable(void)
 816 {
 817         graph_runlevel_changed('S', 0);
 818 }
 819 
 820 static void
 821 graph_post_mu_online(void)
 822 {
 823         graph_runlevel_changed('2', 1);
 824 }
 825 
 826 static void
 827 graph_post_mu_disable(void)
 828 {
 829         graph_runlevel_changed('2', 0);
 830 }
 831 
 832 static void
 833 graph_post_mus_online(void)
 834 {
 835         graph_runlevel_changed('3', 1);
 836 }
 837 
 838 static void
 839 graph_post_mus_disable(void)
 840 {
 841         graph_runlevel_changed('3', 0);
 842 }
 843 
 844 static struct special_vertex_info {
 845         const char      *name;
 846         void            (*start_f)(graph_vertex_t *);
 847         void            (*post_online_f)(void);
 848         void            (*post_disable_f)(void);
 849 } special_vertices[] = {
 850         { CONSOLE_LOGIN_FMRI, graph_clogin_start, NULL, NULL },
 851         { SCF_MILESTONE_SINGLE_USER, graph_su_start,
 852             graph_post_su_online, graph_post_su_disable },
 853         { SCF_MILESTONE_MULTI_USER, NULL,
 854             graph_post_mu_online, graph_post_mu_disable },
 855         { SCF_MILESTONE_MULTI_USER_SERVER, NULL,
 856             graph_post_mus_online, graph_post_mus_disable },
 857         { NULL },
 858 };
 859 
 860 
 861 void
 862 vertex_send_event(graph_vertex_t *v, restarter_event_type_t e)
 863 {
 864         switch (e) {
 865         case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
 866                 assert(v->gv_state == RESTARTER_STATE_UNINIT);
 867 
 868                 MUTEX_LOCK(&st->st_load_lock);
 869                 st->st_load_instances++;
 870                 MUTEX_UNLOCK(&st->st_load_lock);
 871                 break;
 872 
 873         case RESTARTER_EVENT_TYPE_ENABLE:
 874                 log_framework(LOG_DEBUG, "Enabling %s.\n", v->gv_name);
 875                 assert(v->gv_state == RESTARTER_STATE_UNINIT ||
 876                     v->gv_state == RESTARTER_STATE_DISABLED ||
 877                     v->gv_state == RESTARTER_STATE_MAINT);
 878                 break;
 879 
 880         case RESTARTER_EVENT_TYPE_DISABLE:
 881         case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
 882                 log_framework(LOG_DEBUG, "Disabling %s.\n", v->gv_name);
 883                 assert(v->gv_state != RESTARTER_STATE_DISABLED);
 884                 break;
 885 
 886         case RESTARTER_EVENT_TYPE_STOP_RESET:
 887         case RESTARTER_EVENT_TYPE_STOP:
 888                 log_framework(LOG_DEBUG, "Stopping %s.\n", v->gv_name);
 889                 assert(v->gv_state == RESTARTER_STATE_DEGRADED ||
 890                     v->gv_state == RESTARTER_STATE_ONLINE);
 891                 break;
 892 
 893         case RESTARTER_EVENT_TYPE_START:
 894                 log_framework(LOG_DEBUG, "Starting %s.\n", v->gv_name);
 895                 assert(v->gv_state == RESTARTER_STATE_OFFLINE);
 896                 break;
 897 
 898         case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
 899         case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
 900         case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
 901         case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
 902         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
 903         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
 904         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
 905         case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
 906         case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
 907                 break;
 908 
 909         default:
 910 #ifndef NDEBUG
 911                 uu_warn("%s:%d: Bad event %d.\n", __FILE__, __LINE__, e);
 912 #endif
 913                 abort();
 914         }
 915 
 916         restarter_protocol_send_event(v->gv_name, v->gv_restarter_channel, e,
 917             v->gv_reason);
 918 }
 919 
 920 static void
 921 graph_unset_restarter(graph_vertex_t *v)
 922 {
 923         assert(MUTEX_HELD(&dgraph_lock));
 924         assert(v->gv_flags & GV_CONFIGURED);
 925 
 926         vertex_send_event(v, RESTARTER_EVENT_TYPE_REMOVE_INSTANCE);
 927 
 928         if (v->gv_restarter_id != -1) {
 929                 graph_vertex_t *rv;
 930 
 931                 rv = vertex_get_by_id(v->gv_restarter_id);
 932                 graph_remove_edge(v, rv);
 933         }
 934 
 935         v->gv_restarter_id = -1;
 936         v->gv_restarter_channel = NULL;
 937 }
 938 
 939 /*
 940  * Return VERTEX_REMOVED when the vertex passed in argument is deleted from the
 941  * dgraph otherwise return VERTEX_INUSE.
 942  */
 943 static int
 944 free_if_unrefed(graph_vertex_t *v)
 945 {
 946         assert(MUTEX_HELD(&dgraph_lock));
 947 
 948         if (v->gv_refs > 0)
 949                 return (VERTEX_INUSE);
 950 
 951         if (v->gv_type == GVT_SVC &&
 952             uu_list_numnodes(v->gv_dependents) == 0 &&
 953             uu_list_numnodes(v->gv_dependencies) == 0) {
 954                 graph_remove_vertex(v);
 955                 return (VERTEX_REMOVED);
 956         } else if (v->gv_type == GVT_INST &&
 957             (v->gv_flags & GV_CONFIGURED) == 0 &&
 958             uu_list_numnodes(v->gv_dependents) == 1 &&
 959             uu_list_numnodes(v->gv_dependencies) == 0) {
 960                 remove_inst_vertex(v);
 961                 return (VERTEX_REMOVED);
 962         }
 963 
 964         return (VERTEX_INUSE);
 965 }
 966 
 967 static void
 968 delete_depgroup(graph_vertex_t *v)
 969 {
 970         graph_edge_t *e;
 971         graph_vertex_t *dv;
 972 
 973         assert(MUTEX_HELD(&dgraph_lock));
 974         assert(v->gv_type == GVT_GROUP);
 975         assert(uu_list_numnodes(v->gv_dependents) == 0);
 976 
 977         while ((e = uu_list_first(v->gv_dependencies)) != NULL) {
 978                 dv = e->ge_vertex;
 979 
 980                 graph_remove_edge(v, dv);
 981 
 982                 switch (dv->gv_type) {
 983                 case GVT_INST:          /* instance dependency */
 984                 case GVT_SVC:           /* service dependency */
 985                         (void) free_if_unrefed(dv);
 986                         break;
 987 
 988                 case GVT_FILE:          /* file dependency */
 989                         assert(uu_list_numnodes(dv->gv_dependencies) == 0);
 990                         if (uu_list_numnodes(dv->gv_dependents) == 0)
 991                                 graph_remove_vertex(dv);
 992                         break;
 993 
 994                 default:
 995 #ifndef NDEBUG
 996                         uu_warn("%s:%d: Unexpected node type %d", __FILE__,
 997                             __LINE__, dv->gv_type);
 998 #endif
 999                         abort();
1000                 }
1001         }
1002 
1003         graph_remove_vertex(v);
1004 }
1005 
1006 static int
1007 delete_instance_deps_cb(graph_edge_t *e, void **ptrs)
1008 {
1009         graph_vertex_t *v = ptrs[0];
1010         boolean_t delete_restarter_dep = (boolean_t)ptrs[1];
1011         graph_vertex_t *dv;
1012 
1013         dv = e->ge_vertex;
1014 
1015         /*
1016          * We have four possibilities here:
1017          *   - GVT_INST: restarter
1018          *   - GVT_GROUP - GVT_INST: instance dependency
1019          *   - GVT_GROUP - GVT_SVC - GV_INST: service dependency
1020          *   - GVT_GROUP - GVT_FILE: file dependency
1021          */
1022         switch (dv->gv_type) {
1023         case GVT_INST:  /* restarter */
1024                 assert(dv->gv_id == v->gv_restarter_id);
1025                 if (delete_restarter_dep)
1026                         graph_remove_edge(v, dv);
1027                 break;
1028 
1029         case GVT_GROUP: /* pg dependency */
1030                 graph_remove_edge(v, dv);
1031                 delete_depgroup(dv);
1032                 break;
1033 
1034         case GVT_FILE:
1035                 /* These are currently not direct dependencies */
1036 
1037         default:
1038 #ifndef NDEBUG
1039                 uu_warn("%s:%d: Bad vertex type %d.\n", __FILE__, __LINE__,
1040                     dv->gv_type);
1041 #endif
1042                 abort();
1043         }
1044 
1045         return (UU_WALK_NEXT);
1046 }
1047 
1048 static void
1049 delete_instance_dependencies(graph_vertex_t *v, boolean_t delete_restarter_dep)
1050 {
1051         void *ptrs[2];
1052         int r;
1053 
1054         assert(MUTEX_HELD(&dgraph_lock));
1055         assert(v->gv_type == GVT_INST);
1056 
1057         ptrs[0] = v;
1058         ptrs[1] = (void *)delete_restarter_dep;
1059 
1060         r = uu_list_walk(v->gv_dependencies,
1061             (uu_walk_fn_t *)delete_instance_deps_cb, &ptrs, UU_WALK_ROBUST);
1062         assert(r == 0);
1063 }
1064 
1065 /*
1066  * int graph_insert_vertex_unconfigured()
1067  *   Insert a vertex without sending any restarter events. If the vertex
1068  *   already exists or creation is successful, return a pointer to it in *vp.
1069  *
1070  *   If type is not GVT_GROUP, dt can remain unset.
1071  *
1072  *   Returns 0, EEXIST, or EINVAL if the arguments are invalid (i.e., fmri
1073  *   doesn't agree with type, or type doesn't agree with dt).
1074  */
1075 static int
1076 graph_insert_vertex_unconfigured(const char *fmri, gv_type_t type,
1077     depgroup_type_t dt, restarter_error_t rt, graph_vertex_t **vp)
1078 {
1079         int r;
1080         int i;
1081 
1082         assert(MUTEX_HELD(&dgraph_lock));
1083 
1084         switch (type) {
1085         case GVT_SVC:
1086         case GVT_INST:
1087                 if (strncmp(fmri, "svc:", sizeof ("svc:") - 1) != 0)
1088                         return (EINVAL);
1089                 break;
1090 
1091         case GVT_FILE:
1092                 if (strncmp(fmri, "file:", sizeof ("file:") - 1) != 0)
1093                         return (EINVAL);
1094                 break;
1095 
1096         case GVT_GROUP:
1097                 if (dt <= 0 || rt < 0)
1098                         return (EINVAL);
1099                 break;
1100 
1101         default:
1102 #ifndef NDEBUG
1103                 uu_warn("%s:%d: Unknown type %d.\n", __FILE__, __LINE__, type);
1104 #endif
1105                 abort();
1106         }
1107 
1108         *vp = vertex_get_by_name(fmri);
1109         if (*vp != NULL)
1110                 return (EEXIST);
1111 
1112         *vp = graph_add_vertex(fmri);
1113 
1114         (*vp)->gv_type = type;
1115         (*vp)->gv_depgroup = dt;
1116         (*vp)->gv_restart = rt;
1117 
1118         (*vp)->gv_flags = 0;
1119         (*vp)->gv_state = RESTARTER_STATE_NONE;
1120 
1121         for (i = 0; special_vertices[i].name != NULL; ++i) {
1122                 if (strcmp(fmri, special_vertices[i].name) == 0) {
1123                         (*vp)->gv_start_f = special_vertices[i].start_f;
1124                         (*vp)->gv_post_online_f =
1125                             special_vertices[i].post_online_f;
1126                         (*vp)->gv_post_disable_f =
1127                             special_vertices[i].post_disable_f;
1128                         break;
1129                 }
1130         }
1131 
1132         (*vp)->gv_restarter_id = -1;
1133         (*vp)->gv_restarter_channel = 0;
1134 
1135         if (type == GVT_INST) {
1136                 char *sfmri;
1137                 graph_vertex_t *sv;
1138 
1139                 sfmri = inst_fmri_to_svc_fmri(fmri);
1140                 sv = vertex_get_by_name(sfmri);
1141                 if (sv == NULL) {
1142                         r = graph_insert_vertex_unconfigured(sfmri, GVT_SVC, 0,
1143                             0, &sv);
1144                         assert(r == 0);
1145                 }
1146                 startd_free(sfmri, max_scf_fmri_size);
1147 
1148                 graph_add_edge(sv, *vp);
1149         }
1150 
1151         /*
1152          * If this vertex is in the subgraph, mark it as so, for both
1153          * GVT_INST and GVT_SERVICE verteces.
1154          * A GVT_SERVICE vertex can only be in the subgraph if another instance
1155          * depends on it, in which case it's already been added to the graph
1156          * and marked as in the subgraph (by refresh_vertex()).  If a
1157          * GVT_SERVICE vertex was freshly added (by the code above), it means
1158          * that it has no dependents, and cannot be in the subgraph.
1159          * Regardless of this, we still check that gv_flags includes
1160          * GV_INSUBGRAPH in the event that future behavior causes the above
1161          * code to add a GVT_SERVICE vertex which should be in the subgraph.
1162          */
1163 
1164         (*vp)->gv_flags |= (should_be_in_subgraph(*vp)? GV_INSUBGRAPH : 0);
1165 
1166         return (0);
1167 }
1168 
1169 /*
1170  * Returns 0 on success or ELOOP if the dependency would create a cycle.
1171  */
1172 static int
1173 graph_insert_dependency(graph_vertex_t *fv, graph_vertex_t *tv, int **pathp)
1174 {
1175         hrtime_t now;
1176 
1177         assert(MUTEX_HELD(&dgraph_lock));
1178 
1179         /* cycle detection */
1180         now = gethrtime();
1181 
1182         /* Don't follow exclusions. */
1183         if (!(fv->gv_type == GVT_GROUP &&
1184             fv->gv_depgroup == DEPGRP_EXCLUDE_ALL)) {
1185                 *pathp = is_path_to(tv, fv);
1186                 if (*pathp)
1187                         return (ELOOP);
1188         }
1189 
1190         dep_cycle_ns += gethrtime() - now;
1191         ++dep_inserts;
1192         now = gethrtime();
1193 
1194         graph_add_edge(fv, tv);
1195 
1196         dep_insert_ns += gethrtime() - now;
1197 
1198         /* Check if the dependency adds the "to" vertex to the subgraph */
1199         tv->gv_flags |= (should_be_in_subgraph(tv) ? GV_INSUBGRAPH : 0);
1200 
1201         return (0);
1202 }
1203 
1204 static int
1205 inst_running(graph_vertex_t *v)
1206 {
1207         assert(v->gv_type == GVT_INST);
1208 
1209         if (v->gv_state == RESTARTER_STATE_ONLINE ||
1210             v->gv_state == RESTARTER_STATE_DEGRADED)
1211                 return (1);
1212 
1213         return (0);
1214 }
1215 
1216 /*
1217  * The dependency evaluation functions return
1218  *   1 - dependency satisfied
1219  *   0 - dependency unsatisfied
1220  *   -1 - dependency unsatisfiable (without administrator intervention)
1221  *
1222  * The functions also take a boolean satbility argument.  When true, the
1223  * functions may recurse in order to determine satisfiability.
1224  */
1225 static int require_any_satisfied(graph_vertex_t *, boolean_t);
1226 static int dependency_satisfied(graph_vertex_t *, boolean_t);
1227 
1228 /*
1229  * A require_all dependency is unsatisfied if any elements are unsatisfied.  It
1230  * is unsatisfiable if any elements are unsatisfiable.
1231  */
1232 static int
1233 require_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1234 {
1235         graph_edge_t *edge;
1236         int i;
1237         boolean_t any_unsatisfied;
1238 
1239         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1240                 return (1);
1241 
1242         any_unsatisfied = B_FALSE;
1243 
1244         for (edge = uu_list_first(groupv->gv_dependencies);
1245             edge != NULL;
1246             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1247                 i = dependency_satisfied(edge->ge_vertex, satbility);
1248                 if (i == 1)
1249                         continue;
1250 
1251                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1252                     "require_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1253                     edge->ge_vertex->gv_name, i == 0 ? "ed" : "able");
1254 
1255                 if (!satbility)
1256                         return (0);
1257 
1258                 if (i == -1)
1259                         return (-1);
1260 
1261                 any_unsatisfied = B_TRUE;
1262         }
1263 
1264         return (any_unsatisfied ? 0 : 1);
1265 }
1266 
1267 /*
1268  * A require_any dependency is satisfied if any element is satisfied.  It is
1269  * satisfiable if any element is satisfiable.
1270  */
1271 static int
1272 require_any_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1273 {
1274         graph_edge_t *edge;
1275         int s;
1276         boolean_t satisfiable;
1277 
1278         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1279                 return (1);
1280 
1281         satisfiable = B_FALSE;
1282 
1283         for (edge = uu_list_first(groupv->gv_dependencies);
1284             edge != NULL;
1285             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1286                 s = dependency_satisfied(edge->ge_vertex, satbility);
1287 
1288                 if (s == 1)
1289                         return (1);
1290 
1291                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1292                     "require_any(%s): %s is unsatisfi%s.\n",
1293                     groupv->gv_name, edge->ge_vertex->gv_name,
1294                     s == 0 ? "ed" : "able");
1295 
1296                 if (satbility && s == 0)
1297                         satisfiable = B_TRUE;
1298         }
1299 
1300         return (!satbility || satisfiable ? 0 : -1);
1301 }
1302 
1303 /*
1304  * An optional_all dependency only considers elements which are configured,
1305  * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1306  * is unsatisfied.
1307  *
1308  * Offline dependencies which are waiting for a dependency to come online are
1309  * unsatisfied.  Offline dependences which cannot possibly come online
1310  * (unsatisfiable) are always considered satisfied.
1311  */
1312 static int
1313 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1314 {
1315         graph_edge_t *edge;
1316         graph_vertex_t *v;
1317         boolean_t any_qualified;
1318         boolean_t any_unsatisfied;
1319         int i;
1320 
1321         any_qualified = B_FALSE;
1322         any_unsatisfied = B_FALSE;
1323 
1324         for (edge = uu_list_first(groupv->gv_dependencies);
1325             edge != NULL;
1326             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1327                 v = edge->ge_vertex;
1328 
1329                 switch (v->gv_type) {
1330                 case GVT_INST:
1331                         /* Skip missing instances */
1332                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1333                                 continue;
1334 
1335                         if (v->gv_state == RESTARTER_STATE_MAINT)
1336                                 continue;
1337 
1338                         any_qualified = B_TRUE;
1339                         if (v->gv_state == RESTARTER_STATE_OFFLINE) {
1340                                 /*
1341                                  * For offline dependencies, treat unsatisfiable
1342                                  * as satisfied.
1343                                  */
1344                                 i = dependency_satisfied(v, B_TRUE);
1345                                 if (i == -1)
1346                                         i = 1;
1347                         } else if (v->gv_state == RESTARTER_STATE_DISABLED) {
1348                                 /*
1349                                  * If the instance is transitioning out of
1350                                  * disabled the dependency is temporarily
1351                                  * unsatisfied (not unsatisfiable).
1352                                  */
1353                                 i = v->gv_flags & GV_ENABLED ? 0 : 1;
1354                         } else {
1355                                 i = dependency_satisfied(v, satbility);
1356                         }
1357                         break;
1358 
1359                 case GVT_FILE:
1360                         any_qualified = B_TRUE;
1361                         i = dependency_satisfied(v, satbility);
1362 
1363                         break;
1364 
1365                 case GVT_SVC: {
1366                         any_qualified = B_TRUE;
1367                         i = optional_all_satisfied(v, satbility);
1368 
1369                         break;
1370                 }
1371 
1372                 case GVT_GROUP:
1373                 default:
1374 #ifndef NDEBUG
1375                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1376                             __LINE__, v->gv_type);
1377 #endif
1378                         abort();
1379                 }
1380 
1381                 if (i == 1)
1382                         continue;
1383 
1384                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1385                     "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1386                     v->gv_name, i == 0 ? "ed" : "able");
1387 
1388                 if (!satbility)
1389                         return (0);
1390                 if (i == -1)
1391                         return (-1);
1392                 any_unsatisfied = B_TRUE;
1393         }
1394 
1395         if (!any_qualified)
1396                 return (1);
1397 
1398         return (any_unsatisfied ? 0 : 1);
1399 }
1400 
1401 /*
1402  * An exclude_all dependency is unsatisfied if any non-service element is
1403  * satisfied or any service instance which is configured, enabled, and not in
1404  * maintenance is satisfied.  Usually when unsatisfied, it is also
1405  * unsatisfiable.
1406  */
1407 #define LOG_EXCLUDE(u, v)                                               \
1408         log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,                   \
1409             "exclude_all(%s): %s is satisfied.\n",                      \
1410             (u)->gv_name, (v)->gv_name)
1411 
1412 /* ARGSUSED */
1413 static int
1414 exclude_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1415 {
1416         graph_edge_t *edge, *e2;
1417         graph_vertex_t *v, *v2;
1418 
1419         for (edge = uu_list_first(groupv->gv_dependencies);
1420             edge != NULL;
1421             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1422                 v = edge->ge_vertex;
1423 
1424                 switch (v->gv_type) {
1425                 case GVT_INST:
1426                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1427                                 continue;
1428 
1429                         switch (v->gv_state) {
1430                         case RESTARTER_STATE_ONLINE:
1431                         case RESTARTER_STATE_DEGRADED:
1432                         case RESTARTER_STATE_OFFLINE:
1433                                 LOG_EXCLUDE(groupv, v);
1434                                 return (v->gv_flags & GV_TODISABLE ? 0 : -1);
1435 
1436                         case RESTARTER_STATE_DISABLED:
1437                         case RESTARTER_STATE_UNINIT:
1438                                 if (v->gv_flags & GV_ENABLED) {
1439                                         LOG_EXCLUDE(groupv, v);
1440                                         return (-1);
1441                                 }
1442                                 /* FALLTHROUGH */
1443                         case RESTARTER_STATE_MAINT:
1444                                 continue;
1445 
1446                         default:
1447 #ifndef NDEBUG
1448                                 uu_warn("%s:%d: Unexpected vertex state %d.\n",
1449                                     __FILE__, __LINE__, v->gv_state);
1450 #endif
1451                                 abort();
1452                         }
1453                         /* NOTREACHED */
1454 
1455                 case GVT_SVC:
1456                         return (exclude_all_satisfied(v, satbility));
1457 
1458                 case GVT_FILE:
1459                         if (!file_ready(v))
1460                                 continue;
1461                         LOG_EXCLUDE(groupv, v);
1462                         return (-1);
1463 
1464                 case GVT_GROUP:
1465                 default:
1466 #ifndef NDEBUG
1467                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1468                             __LINE__, v->gv_type);
1469 #endif
1470                         abort();
1471                 }
1472         }
1473 
1474         return (1);
1475 }
1476 
1477 /*
1478  * int instance_satisfied()
1479  *   Determine if all the dependencies are satisfied for the supplied instance
1480  *   vertex. Return 1 if they are, 0 if they aren't, and -1 if they won't be
1481  *   without administrator intervention.
1482  */
1483 static int
1484 instance_satisfied(graph_vertex_t *v, boolean_t satbility)
1485 {
1486         graph_edge_t *e;
1487         boolean_t any_unsatisfied;
1488         boolean_t any_unsatisfiable;
1489 
1490         assert(v->gv_type == GVT_INST);
1491         assert(!inst_running(v));
1492 
1493         any_unsatisfied = B_FALSE;
1494         any_unsatisfiable = B_FALSE;
1495 
1496         /*
1497          * Walk the dependents of the given vertex looking for any exclude_all
1498          * dependency groups with running instances.  If they are all going to
1499          * be disabled consider the dependency unsatisfied, not unsatisfaible.
1500          */
1501         for (e = uu_list_first(v->gv_dependents); e != NULL;
1502             e = uu_list_next(v->gv_dependents, e)) {
1503                 graph_edge_t *e2;
1504 
1505                 if (e->ge_vertex->gv_type != GVT_GROUP ||
1506                     e->ge_vertex->gv_depgroup != DEPGRP_EXCLUDE_ALL)
1507                         continue;
1508 
1509                 for (e2 = uu_list_first(e->ge_vertex->gv_dependents);
1510                     e2 != NULL;
1511                     e2 = uu_list_next(e->ge_vertex->gv_dependents, e2)) {
1512 
1513                         if (!inst_running(e2->ge_vertex))
1514                                 continue;
1515 
1516                         if (e2->ge_vertex->gv_flags & GV_TODISABLE)
1517                                 any_unsatisfied = B_TRUE;
1518                         else
1519                                 any_unsatisfiable = B_TRUE;
1520                 }
1521         }
1522 
1523         if (any_unsatisfied || any_unsatisfiable)
1524                 return (any_unsatisfiable ? -1 : 0);
1525 
1526         return (require_all_satisfied(v, satbility));
1527 }
1528 
1529 /*
1530  * Decide whether v can satisfy a dependency.  v can either be a child of
1531  * a group vertex, or of an instance vertex.
1532  */
1533 static int
1534 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1535 {
1536         switch (v->gv_type) {
1537         case GVT_INST:
1538                 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1539                         if (v->gv_flags & GV_DEATHROW) {
1540                                 /*
1541                                  * A dependency on an instance with GV_DEATHROW
1542                                  * flag is always considered as satisfied.
1543                                  */
1544                                 return (1);
1545                         }
1546                         return (-1);
1547                 }
1548 
1549                 /*
1550                  * Any vertex with the GV_TODISABLE flag set is guaranteed
1551                  * to have its dependencies unsatisfiable.  Any vertex with
1552                  * GV_TOOFFLINE may be satisfied after it transitions.
1553                  */
1554                 if (v->gv_flags & GV_TODISABLE)
1555                         return (-1);
1556                 if (v->gv_flags & GV_TOOFFLINE)
1557                         return (0);
1558 
1559                 switch (v->gv_state) {
1560                 case RESTARTER_STATE_ONLINE:
1561                 case RESTARTER_STATE_DEGRADED:
1562                         return (1);
1563 
1564                 case RESTARTER_STATE_OFFLINE:
1565                         if (!satbility)
1566                                 return (0);
1567                         return (instance_satisfied(v, satbility) != -1 ?
1568                             0 : -1);
1569 
1570                 case RESTARTER_STATE_DISABLED:
1571                 case RESTARTER_STATE_MAINT:
1572                         return (-1);
1573 
1574                 case RESTARTER_STATE_UNINIT:
1575                         return (0);
1576 
1577                 default:
1578 #ifndef NDEBUG
1579                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
1580                             __FILE__, __LINE__, v->gv_state);
1581 #endif
1582                         abort();
1583                         /* NOTREACHED */
1584                 }
1585 
1586         case GVT_SVC:
1587                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1588                         return (-1);
1589                 return (require_any_satisfied(v, satbility));
1590 
1591         case GVT_FILE:
1592                 /* i.e., we assume files will not be automatically generated */
1593                 return (file_ready(v) ? 1 : -1);
1594 
1595         case GVT_GROUP:
1596                 break;
1597 
1598         default:
1599 #ifndef NDEBUG
1600                 uu_warn("%s:%d: Unexpected node type %d.\n", __FILE__, __LINE__,
1601                     v->gv_type);
1602 #endif
1603                 abort();
1604                 /* NOTREACHED */
1605         }
1606 
1607         switch (v->gv_depgroup) {
1608         case DEPGRP_REQUIRE_ANY:
1609                 return (require_any_satisfied(v, satbility));
1610 
1611         case DEPGRP_REQUIRE_ALL:
1612                 return (require_all_satisfied(v, satbility));
1613 
1614         case DEPGRP_OPTIONAL_ALL:
1615                 return (optional_all_satisfied(v, satbility));
1616 
1617         case DEPGRP_EXCLUDE_ALL:
1618                 return (exclude_all_satisfied(v, satbility));
1619 
1620         default:
1621 #ifndef NDEBUG
1622                 uu_warn("%s:%d: Unknown dependency grouping %d.\n", __FILE__,
1623                     __LINE__, v->gv_depgroup);
1624 #endif
1625                 abort();
1626         }
1627 }
1628 
1629 void
1630 graph_start_if_satisfied(graph_vertex_t *v)
1631 {
1632         if (v->gv_state == RESTARTER_STATE_OFFLINE &&
1633             instance_satisfied(v, B_FALSE) == 1) {
1634                 if (v->gv_start_f == NULL)
1635                         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
1636                 else
1637                         v->gv_start_f(v);
1638         }
1639 }
1640 
1641 /*
1642  * propagate_satbility()
1643  *
1644  * This function is used when the given vertex changes state in such a way that
1645  * one of its dependents may become unsatisfiable.  This happens when an
1646  * instance transitions between offline -> online, or from !running ->
1647  * maintenance, as well as when an instance is removed from the graph.
1648  *
1649  * We have to walk all the dependents, since optional_all dependencies several
1650  * levels up could become (un)satisfied, instead of unsatisfiable.  For example,
1651  *
1652  *      +-----+  optional_all  +-----+  require_all  +-----+
1653  *      |  A  |--------------->|  B  |-------------->|  C  |
1654  *      +-----+                +-----+               +-----+
1655  *
1656  *                                              offline -> maintenance
1657  *
1658  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1659  * an optional dependency, what was previously an unsatisfiable situation is now
1660  * satisfied (B will never come online, even though its state hasn't changed).
1661  *
1662  * Note that it's not necessary to continue examining dependents after reaching
1663  * an optional_all dependency.  It's not possible for an optional_all dependency
1664  * to change satisfiability without also coming online, in which case we get a
1665  * start event and propagation continues naturally.  However, it does no harm to
1666  * continue propagating satisfiability (as it is a relatively rare event), and
1667  * keeps the walker code simple and generic.
1668  */
1669 /*ARGSUSED*/
1670 static int
1671 satbility_cb(graph_vertex_t *v, void *arg)
1672 {
1673         if (v->gv_flags & GV_TOOFFLINE)
1674                 return (UU_WALK_NEXT);
1675 
1676         if (v->gv_type == GVT_INST)
1677                 graph_start_if_satisfied(v);
1678 
1679         return (UU_WALK_NEXT);
1680 }
1681 
1682 static void
1683 propagate_satbility(graph_vertex_t *v)
1684 {
1685         graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1686 }
1687 
1688 static void propagate_stop(graph_vertex_t *, void *);
1689 
1690 /*
1691  * propagate_start()
1692  *
1693  * This function is used to propagate a start event to the dependents of the
1694  * given vertex.  Any dependents that are offline but have their dependencies
1695  * satisfied are started.  Any dependents that are online and have restart_on
1696  * set to "restart" or "refresh" are restarted because their dependencies have
1697  * just changed.  This only happens with optional_all dependencies.
1698  */
1699 static void
1700 propagate_start(graph_vertex_t *v, void *arg)
1701 {
1702         restarter_error_t err = (restarter_error_t)arg;
1703 
1704         if (v->gv_flags & GV_TOOFFLINE)
1705                 return;
1706 
1707         switch (v->gv_type) {
1708         case GVT_INST:
1709                 /* Restarter */
1710                 if (inst_running(v)) {
1711                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1712                                 vertex_send_event(v,
1713                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1714                         }
1715                 } else {
1716                         graph_start_if_satisfied(v);
1717                 }
1718                 break;
1719 
1720         case GVT_GROUP:
1721                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1722                         graph_walk_dependents(v, propagate_stop,
1723                             (void *)RERR_RESTART);
1724                         break;
1725                 }
1726                 err = v->gv_restart;
1727                 /* FALLTHROUGH */
1728 
1729         case GVT_SVC:
1730                 graph_walk_dependents(v, propagate_start, (void *)err);
1731                 break;
1732 
1733         case GVT_FILE:
1734 #ifndef NDEBUG
1735                 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1736                     __FILE__, __LINE__);
1737 #endif
1738                 abort();
1739                 /* NOTREACHED */
1740 
1741         default:
1742 #ifndef NDEBUG
1743                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1744                     v->gv_type);
1745 #endif
1746                 abort();
1747         }
1748 }
1749 
1750 /*
1751  * propagate_stop()
1752  *
1753  * This function is used to propagate a stop event to the dependents of the
1754  * given vertex.  Any dependents that are online (or in degraded state) with
1755  * the restart_on property set to "restart" or "refresh" will be stopped as
1756  * their dependencies have just changed, propagate_start() will start them
1757  * again once their dependencies have been re-satisfied.
1758  */
1759 static void
1760 propagate_stop(graph_vertex_t *v, void *arg)
1761 {
1762         restarter_error_t err = (restarter_error_t)arg;
1763 
1764         if (v->gv_flags & GV_TOOFFLINE)
1765                 return;
1766 
1767         switch (v->gv_type) {
1768         case GVT_INST:
1769                 /* Restarter */
1770                 if (err > RERR_NONE && inst_running(v)) {
1771                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1772                                 vertex_send_event(v,
1773                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1774                         } else {
1775                                 vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1776                         }
1777                 }
1778                 break;
1779 
1780         case GVT_SVC:
1781                 graph_walk_dependents(v, propagate_stop, arg);
1782                 break;
1783 
1784         case GVT_FILE:
1785 #ifndef NDEBUG
1786                 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1787                     __FILE__, __LINE__);
1788 #endif
1789                 abort();
1790                 /* NOTREACHED */
1791 
1792         case GVT_GROUP:
1793                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1794                         graph_walk_dependents(v, propagate_start, NULL);
1795                         break;
1796                 }
1797 
1798                 if (err == RERR_NONE || err > v->gv_restart)
1799                         break;
1800 
1801                 graph_walk_dependents(v, propagate_stop, arg);
1802                 break;
1803 
1804         default:
1805 #ifndef NDEBUG
1806                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1807                     v->gv_type);
1808 #endif
1809                 abort();
1810         }
1811 }
1812 
1813 void
1814 offline_vertex(graph_vertex_t *v)
1815 {
1816         scf_handle_t *h = libscf_handle_create_bound_loop();
1817         scf_instance_t *scf_inst = safe_scf_instance_create(h);
1818         scf_propertygroup_t *pg = safe_scf_pg_create(h);
1819         restarter_instance_state_t state, next_state;
1820         int r;
1821 
1822         assert(v->gv_type == GVT_INST);
1823 
1824         if (scf_inst == NULL)
1825                 bad_error("safe_scf_instance_create", scf_error());
1826         if (pg == NULL)
1827                 bad_error("safe_scf_pg_create", scf_error());
1828 
1829         /* if the vertex is already going offline, return */
1830 rep_retry:
1831         if (scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, scf_inst, NULL,
1832             NULL, SCF_DECODE_FMRI_EXACT) != 0) {
1833                 switch (scf_error()) {
1834                 case SCF_ERROR_CONNECTION_BROKEN:
1835                         libscf_handle_rebind(h);
1836                         goto rep_retry;
1837 
1838                 case SCF_ERROR_NOT_FOUND:
1839                         scf_pg_destroy(pg);
1840                         scf_instance_destroy(scf_inst);
1841                         (void) scf_handle_unbind(h);
1842                         scf_handle_destroy(h);
1843                         return;
1844                 }
1845                 uu_die("Can't decode FMRI %s: %s\n", v->gv_name,
1846                     scf_strerror(scf_error()));
1847         }
1848 
1849         r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
1850         if (r != 0) {
1851                 switch (scf_error()) {
1852                 case SCF_ERROR_CONNECTION_BROKEN:
1853                         libscf_handle_rebind(h);
1854                         goto rep_retry;
1855 
1856                 case SCF_ERROR_NOT_SET:
1857                 case SCF_ERROR_NOT_FOUND:
1858                         scf_pg_destroy(pg);
1859                         scf_instance_destroy(scf_inst);
1860                         (void) scf_handle_unbind(h);
1861                         scf_handle_destroy(h);
1862                         return;
1863 
1864                 default:
1865                         bad_error("scf_instance_get_pg", scf_error());
1866                 }
1867         } else {
1868                 r = libscf_read_states(pg, &state, &next_state);
1869                 if (r == 0 && (next_state == RESTARTER_STATE_OFFLINE ||
1870                     next_state == RESTARTER_STATE_DISABLED)) {
1871                         log_framework(LOG_DEBUG,
1872                             "%s: instance is already going down.\n",
1873                             v->gv_name);
1874                         scf_pg_destroy(pg);
1875                         scf_instance_destroy(scf_inst);
1876                         (void) scf_handle_unbind(h);
1877                         scf_handle_destroy(h);
1878                         return;
1879                 }
1880         }
1881 
1882         scf_pg_destroy(pg);
1883         scf_instance_destroy(scf_inst);
1884         (void) scf_handle_unbind(h);
1885         scf_handle_destroy(h);
1886 
1887         vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP_RESET);
1888 }
1889 
1890 /*
1891  * void graph_enable_by_vertex()
1892  *   If admin is non-zero, this is an administrative request for change
1893  *   of the enabled property.  Thus, send the ADMIN_DISABLE rather than
1894  *   a plain DISABLE restarter event.
1895  */
1896 void
1897 graph_enable_by_vertex(graph_vertex_t *vertex, int enable, int admin)
1898 {
1899         graph_vertex_t *v;
1900         int r;
1901 
1902         assert(MUTEX_HELD(&dgraph_lock));
1903         assert((vertex->gv_flags & GV_CONFIGURED));
1904 
1905         vertex->gv_flags = (vertex->gv_flags & ~GV_ENABLED) |
1906             (enable ? GV_ENABLED : 0);
1907 
1908         if (enable) {
1909                 if (vertex->gv_state != RESTARTER_STATE_OFFLINE &&
1910                     vertex->gv_state != RESTARTER_STATE_DEGRADED &&
1911                     vertex->gv_state != RESTARTER_STATE_ONLINE) {
1912                         /*
1913                          * In case the vertex was notified to go down,
1914                          * but now can return online, clear the _TOOFFLINE
1915                          * and _TODISABLE flags.
1916                          */
1917                         vertex->gv_flags &= ~GV_TOOFFLINE;
1918                         vertex->gv_flags &= ~GV_TODISABLE;
1919 
1920                         vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ENABLE);
1921                 }
1922 
1923                 /*
1924                  * Wait for state update from restarter before sending _START or
1925                  * _STOP.
1926                  */
1927 
1928                 return;
1929         }
1930 
1931         if (vertex->gv_state == RESTARTER_STATE_DISABLED)
1932                 return;
1933 
1934         if (!admin) {
1935                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_DISABLE);
1936 
1937                 /*
1938                  * Wait for state update from restarter before sending _START or
1939                  * _STOP.
1940                  */
1941 
1942                 return;
1943         }
1944 
1945         /*
1946          * If it is a DISABLE event requested by the administrator then we are
1947          * offlining the dependents first.
1948          */
1949 
1950         /*
1951          * Set GV_TOOFFLINE for the services we are offlining. We cannot
1952          * clear the GV_TOOFFLINE bits from all the services because
1953          * other DISABLE events might be handled at the same time.
1954          */
1955         vertex->gv_flags |= GV_TOOFFLINE;
1956 
1957         /* remember which vertex to disable... */
1958         vertex->gv_flags |= GV_TODISABLE;
1959 
1960         log_framework(LOG_DEBUG, "Marking in-subtree vertices before "
1961             "disabling %s.\n", vertex->gv_name);
1962 
1963         /* set GV_TOOFFLINE for its dependents */
1964         r = uu_list_walk(vertex->gv_dependents, (uu_walk_fn_t *)mark_subtree,
1965             NULL, 0);
1966         assert(r == 0);
1967 
1968         /* disable the instance now if there is nothing else to offline */
1969         if (insubtree_dependents_down(vertex) == B_TRUE) {
1970                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
1971                 return;
1972         }
1973 
1974         /*
1975          * This loop is similar to the one used for the graph reversal shutdown
1976          * and could be improved in term of performance for the subtree reversal
1977          * disable case.
1978          */
1979         for (v = uu_list_first(dgraph); v != NULL;
1980             v = uu_list_next(dgraph, v)) {
1981                 /* skip the vertex we are disabling for now */
1982                 if (v == vertex)
1983                         continue;
1984 
1985                 if (v->gv_type != GVT_INST ||
1986                     (v->gv_flags & GV_CONFIGURED) == 0 ||
1987                     (v->gv_flags & GV_ENABLED) == 0 ||
1988                     (v->gv_flags & GV_TOOFFLINE) == 0)
1989                         continue;
1990 
1991                 if ((v->gv_state != RESTARTER_STATE_ONLINE) &&
1992                     (v->gv_state != RESTARTER_STATE_DEGRADED)) {
1993                         /* continue if there is nothing to offline */
1994                         continue;
1995                 }
1996 
1997                 /*
1998                  * Instances which are up need to come down before we're
1999                  * done, but we can only offline the leaves here. An
2000                  * instance is a leaf when all its dependents are down.
2001                  */
2002                 if (insubtree_dependents_down(v) == B_TRUE) {
2003                         log_framework(LOG_DEBUG, "Offlining in-subtree "
2004                             "instance %s for %s.\n",
2005                             v->gv_name, vertex->gv_name);
2006                         offline_vertex(v);
2007                 }
2008         }
2009 }
2010 
2011 static int configure_vertex(graph_vertex_t *, scf_instance_t *);
2012 
2013 /*
2014  * Set the restarter for v to fmri_arg.  That is, make sure a vertex for
2015  * fmri_arg exists, make v depend on it, and send _ADD_INSTANCE for v.  If
2016  * v is already configured and fmri_arg indicates the current restarter, do
2017  * nothing.  If v is configured and fmri_arg is a new restarter, delete v's
2018  * dependency on the restarter, send _REMOVE_INSTANCE for v, and set the new
2019  * restarter.  Returns 0 on success, EINVAL if the FMRI is invalid,
2020  * ECONNABORTED if the repository connection is broken, and ELOOP
2021  * if the dependency would create a cycle.  In the last case, *pathp will
2022  * point to a -1-terminated array of ids which compose the path from v to
2023  * restarter_fmri.
2024  */
2025 int
2026 graph_change_restarter(graph_vertex_t *v, const char *fmri_arg, scf_handle_t *h,
2027     int **pathp)
2028 {
2029         char *restarter_fmri = NULL;
2030         graph_vertex_t *rv;
2031         int err;
2032         int id;
2033 
2034         assert(MUTEX_HELD(&dgraph_lock));
2035 
2036         if (fmri_arg[0] != '\0') {
2037                 err = fmri_canonify(fmri_arg, &restarter_fmri, B_TRUE);
2038                 if (err != 0) {
2039                         assert(err == EINVAL);
2040                         return (err);
2041                 }
2042         }
2043 
2044         if (restarter_fmri == NULL ||
2045             strcmp(restarter_fmri, SCF_SERVICE_STARTD) == 0) {
2046                 if (v->gv_flags & GV_CONFIGURED) {
2047                         if (v->gv_restarter_id == -1) {
2048                                 if (restarter_fmri != NULL)
2049                                         startd_free(restarter_fmri,
2050                                             max_scf_fmri_size);
2051                                 return (0);
2052                         }
2053 
2054                         graph_unset_restarter(v);
2055                 }
2056 
2057                 /* Master restarter, nothing to do. */
2058                 v->gv_restarter_id = -1;
2059                 v->gv_restarter_channel = NULL;
2060                 vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2061                 return (0);
2062         }
2063 
2064         if (v->gv_flags & GV_CONFIGURED) {
2065                 id = dict_lookup_byname(restarter_fmri);
2066                 if (id != -1 && v->gv_restarter_id == id) {
2067                         startd_free(restarter_fmri, max_scf_fmri_size);
2068                         return (0);
2069                 }
2070 
2071                 graph_unset_restarter(v);
2072         }
2073 
2074         err = graph_insert_vertex_unconfigured(restarter_fmri, GVT_INST, 0,
2075             RERR_NONE, &rv);
2076         startd_free(restarter_fmri, max_scf_fmri_size);
2077         assert(err == 0 || err == EEXIST);
2078 
2079         if (rv->gv_delegate_initialized == 0) {
2080                 if ((rv->gv_delegate_channel = restarter_protocol_init_delegate(
2081                     rv->gv_name)) == NULL)
2082                         return (EINVAL);
2083                 rv->gv_delegate_initialized = 1;
2084         }
2085         v->gv_restarter_id = rv->gv_id;
2086         v->gv_restarter_channel = rv->gv_delegate_channel;
2087 
2088         err = graph_insert_dependency(v, rv, pathp);
2089         if (err != 0) {
2090                 assert(err == ELOOP);
2091                 return (ELOOP);
2092         }
2093 
2094         vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2095 
2096         if (!(rv->gv_flags & GV_CONFIGURED)) {
2097                 scf_instance_t *inst;
2098 
2099                 err = libscf_fmri_get_instance(h, rv->gv_name, &inst);
2100                 switch (err) {
2101                 case 0:
2102                         err = configure_vertex(rv, inst);
2103                         scf_instance_destroy(inst);
2104                         switch (err) {
2105                         case 0:
2106                         case ECANCELED:
2107                                 break;
2108 
2109                         case ECONNABORTED:
2110                                 return (ECONNABORTED);
2111 
2112                         default:
2113                                 bad_error("configure_vertex", err);
2114                         }
2115                         break;
2116 
2117                 case ECONNABORTED:
2118                         return (ECONNABORTED);
2119 
2120                 case ENOENT:
2121                         break;
2122 
2123                 case ENOTSUP:
2124                         /*
2125                          * The fmri doesn't specify an instance - translate
2126                          * to EINVAL.
2127                          */
2128                         return (EINVAL);
2129 
2130                 case EINVAL:
2131                 default:
2132                         bad_error("libscf_fmri_get_instance", err);
2133                 }
2134         }
2135 
2136         return (0);
2137 }
2138 
2139 
2140 /*
2141  * Add all of the instances of the service named by fmri to the graph.
2142  * Returns
2143  *   0 - success
2144  *   ENOENT - service indicated by fmri does not exist
2145  *
2146  * In both cases *reboundp will be B_TRUE if the handle was rebound, or B_FALSE
2147  * otherwise.
2148  */
2149 static int
2150 add_service(const char *fmri, scf_handle_t *h, boolean_t *reboundp)
2151 {
2152         scf_service_t *svc;
2153         scf_instance_t *inst;
2154         scf_iter_t *iter;
2155         char *inst_fmri;
2156         int ret, r;
2157 
2158         *reboundp = B_FALSE;
2159 
2160         svc = safe_scf_service_create(h);
2161         inst = safe_scf_instance_create(h);
2162         iter = safe_scf_iter_create(h);
2163         inst_fmri = startd_alloc(max_scf_fmri_size);
2164 
2165 rebound:
2166         if (scf_handle_decode_fmri(h, fmri, NULL, svc, NULL, NULL, NULL,
2167             SCF_DECODE_FMRI_EXACT) != 0) {
2168                 switch (scf_error()) {
2169                 case SCF_ERROR_CONNECTION_BROKEN:
2170                 default:
2171                         libscf_handle_rebind(h);
2172                         *reboundp = B_TRUE;
2173                         goto rebound;
2174 
2175                 case SCF_ERROR_NOT_FOUND:
2176                         ret = ENOENT;
2177                         goto out;
2178 
2179                 case SCF_ERROR_INVALID_ARGUMENT:
2180                 case SCF_ERROR_CONSTRAINT_VIOLATED:
2181                 case SCF_ERROR_NOT_BOUND:
2182                 case SCF_ERROR_HANDLE_MISMATCH:
2183                         bad_error("scf_handle_decode_fmri", scf_error());
2184                 }
2185         }
2186 
2187         if (scf_iter_service_instances(iter, svc) != 0) {
2188                 switch (scf_error()) {
2189                 case SCF_ERROR_CONNECTION_BROKEN:
2190                 default:
2191                         libscf_handle_rebind(h);
2192                         *reboundp = B_TRUE;
2193                         goto rebound;
2194 
2195                 case SCF_ERROR_DELETED:
2196                         ret = ENOENT;
2197                         goto out;
2198 
2199                 case SCF_ERROR_HANDLE_MISMATCH:
2200                 case SCF_ERROR_NOT_BOUND:
2201                 case SCF_ERROR_NOT_SET:
2202                         bad_error("scf_iter_service_instances", scf_error());
2203                 }
2204         }
2205 
2206         for (;;) {
2207                 r = scf_iter_next_instance(iter, inst);
2208                 if (r == 0)
2209                         break;
2210                 if (r != 1) {
2211                         switch (scf_error()) {
2212                         case SCF_ERROR_CONNECTION_BROKEN:
2213                         default:
2214                                 libscf_handle_rebind(h);
2215                                 *reboundp = B_TRUE;
2216                                 goto rebound;
2217 
2218                         case SCF_ERROR_DELETED:
2219                                 ret = ENOENT;
2220                                 goto out;
2221 
2222                         case SCF_ERROR_HANDLE_MISMATCH:
2223                         case SCF_ERROR_NOT_BOUND:
2224                         case SCF_ERROR_NOT_SET:
2225                         case SCF_ERROR_INVALID_ARGUMENT:
2226                                 bad_error("scf_iter_next_instance",
2227                                     scf_error());
2228                         }
2229                 }
2230 
2231                 if (scf_instance_to_fmri(inst, inst_fmri, max_scf_fmri_size) <
2232                     0) {
2233                         switch (scf_error()) {
2234                         case SCF_ERROR_CONNECTION_BROKEN:
2235                                 libscf_handle_rebind(h);
2236                                 *reboundp = B_TRUE;
2237                                 goto rebound;
2238 
2239                         case SCF_ERROR_DELETED:
2240                                 continue;
2241 
2242                         case SCF_ERROR_NOT_BOUND:
2243                         case SCF_ERROR_NOT_SET:
2244                                 bad_error("scf_instance_to_fmri", scf_error());
2245                         }
2246                 }
2247 
2248                 r = dgraph_add_instance(inst_fmri, inst, B_FALSE);
2249                 switch (r) {
2250                 case 0:
2251                 case ECANCELED:
2252                         break;
2253 
2254                 case EEXIST:
2255                         continue;
2256 
2257                 case ECONNABORTED:
2258                         libscf_handle_rebind(h);
2259                         *reboundp = B_TRUE;
2260                         goto rebound;
2261 
2262                 case EINVAL:
2263                 default:
2264                         bad_error("dgraph_add_instance", r);
2265                 }
2266         }
2267 
2268         ret = 0;
2269 
2270 out:
2271         startd_free(inst_fmri, max_scf_fmri_size);
2272         scf_iter_destroy(iter);
2273         scf_instance_destroy(inst);
2274         scf_service_destroy(svc);
2275         return (ret);
2276 }
2277 
2278 struct depfmri_info {
2279         graph_vertex_t  *v;             /* GVT_GROUP vertex */
2280         gv_type_t       type;           /* type of dependency */
2281         const char      *inst_fmri;     /* FMRI of parental GVT_INST vert. */
2282         const char      *pg_name;       /* Name of dependency pg */
2283         scf_handle_t    *h;
2284         int             err;            /* return error code */
2285         int             **pathp;        /* return circular dependency path */
2286 };
2287 
2288 /*
2289  * Find or create a vertex for fmri and make info->v depend on it.
2290  * Returns
2291  *   0 - success
2292  *   nonzero - failure
2293  *
2294  * On failure, sets info->err to
2295  *   EINVAL - fmri is invalid
2296  *            fmri does not match info->type
2297  *   ELOOP - Adding the dependency creates a circular dependency.  *info->pathp
2298  *           will point to an array of the ids of the members of the cycle.
2299  *   ECONNABORTED - repository connection was broken
2300  *   ECONNRESET - succeeded, but repository connection was reset
2301  */
2302 static int
2303 process_dependency_fmri(const char *fmri, struct depfmri_info *info)
2304 {
2305         int err;
2306         graph_vertex_t *depgroup_v, *v;
2307         char *fmri_copy, *cfmri;
2308         size_t fmri_copy_sz;
2309         const char *scope, *service, *instance, *pg;
2310         scf_instance_t *inst;
2311         boolean_t rebound;
2312 
2313         assert(MUTEX_HELD(&dgraph_lock));
2314 
2315         /* Get or create vertex for FMRI */
2316         depgroup_v = info->v;
2317 
2318         if (strncmp(fmri, "file:", sizeof ("file:") - 1) == 0) {
2319                 if (info->type != GVT_FILE) {
2320                         log_framework(LOG_NOTICE,
2321                             "FMRI \"%s\" is not allowed for the \"%s\" "
2322                             "dependency's type of instance %s.\n", fmri,
2323                             info->pg_name, info->inst_fmri);
2324                         return (info->err = EINVAL);
2325                 }
2326 
2327                 err = graph_insert_vertex_unconfigured(fmri, info->type, 0,
2328                     RERR_NONE, &v);
2329                 switch (err) {
2330                 case 0:
2331                         break;
2332 
2333                 case EEXIST:
2334                         assert(v->gv_type == GVT_FILE);
2335                         break;
2336 
2337                 case EINVAL:            /* prevented above */
2338                 default:
2339                         bad_error("graph_insert_vertex_unconfigured", err);
2340                 }
2341         } else {
2342                 if (info->type != GVT_INST) {
2343                         log_framework(LOG_NOTICE,
2344                             "FMRI \"%s\" is not allowed for the \"%s\" "
2345                             "dependency's type of instance %s.\n", fmri,
2346                             info->pg_name, info->inst_fmri);
2347                         return (info->err = EINVAL);
2348                 }
2349 
2350                 /*
2351                  * We must canonify fmri & add a vertex for it.
2352                  */
2353                 fmri_copy_sz = strlen(fmri) + 1;
2354                 fmri_copy = startd_alloc(fmri_copy_sz);
2355                 (void) strcpy(fmri_copy, fmri);
2356 
2357                 /* Determine if the FMRI is a property group or instance */
2358                 if (scf_parse_svc_fmri(fmri_copy, &scope, &service,
2359                     &instance, &pg, NULL) != 0) {
2360                         startd_free(fmri_copy, fmri_copy_sz);
2361                         log_framework(LOG_NOTICE,
2362                             "Dependency \"%s\" of %s has invalid FMRI "
2363                             "\"%s\".\n", info->pg_name, info->inst_fmri,
2364                             fmri);
2365                         return (info->err = EINVAL);
2366                 }
2367 
2368                 if (service == NULL || pg != NULL) {
2369                         startd_free(fmri_copy, fmri_copy_sz);
2370                         log_framework(LOG_NOTICE,
2371                             "Dependency \"%s\" of %s does not designate a "
2372                             "service or instance.\n", info->pg_name,
2373                             info->inst_fmri);
2374                         return (info->err = EINVAL);
2375                 }
2376 
2377                 if (scope == NULL || strcmp(scope, SCF_SCOPE_LOCAL) == 0) {
2378                         cfmri = uu_msprintf("svc:/%s%s%s",
2379                             service, instance ? ":" : "", instance ? instance :
2380                             "");
2381                 } else {
2382                         cfmri = uu_msprintf("svc://%s/%s%s%s",
2383                             scope, service, instance ? ":" : "", instance ?
2384                             instance : "");
2385                 }
2386 
2387                 startd_free(fmri_copy, fmri_copy_sz);
2388 
2389                 err = graph_insert_vertex_unconfigured(cfmri, instance ?
2390                     GVT_INST : GVT_SVC, instance ? 0 : DEPGRP_REQUIRE_ANY,
2391                     RERR_NONE, &v);
2392                 uu_free(cfmri);
2393                 switch (err) {
2394                 case 0:
2395                         break;
2396 
2397                 case EEXIST:
2398                         /* Verify v. */
2399                         if (instance != NULL)
2400                                 assert(v->gv_type == GVT_INST);
2401                         else
2402                                 assert(v->gv_type == GVT_SVC);
2403                         break;
2404 
2405                 default:
2406                         bad_error("graph_insert_vertex_unconfigured", err);
2407                 }
2408         }
2409 
2410         /* Add dependency from depgroup_v to new vertex */
2411         info->err = graph_insert_dependency(depgroup_v, v, info->pathp);
2412         switch (info->err) {
2413         case 0:
2414                 break;
2415 
2416         case ELOOP:
2417                 return (ELOOP);
2418 
2419         default:
2420                 bad_error("graph_insert_dependency", info->err);
2421         }
2422 
2423         /* This must be after we insert the dependency, to avoid looping. */
2424         switch (v->gv_type) {
2425         case GVT_INST:
2426                 if ((v->gv_flags & GV_CONFIGURED) != 0)
2427                         break;
2428 
2429                 inst = safe_scf_instance_create(info->h);
2430 
2431                 rebound = B_FALSE;
2432 
2433 rebound:
2434                 err = libscf_lookup_instance(v->gv_name, inst);
2435                 switch (err) {
2436                 case 0:
2437                         err = configure_vertex(v, inst);
2438                         switch (err) {
2439                         case 0:
2440                         case ECANCELED:
2441                                 break;
2442 
2443                         case ECONNABORTED:
2444                                 libscf_handle_rebind(info->h);
2445                                 rebound = B_TRUE;
2446                                 goto rebound;
2447 
2448                         default:
2449                                 bad_error("configure_vertex", err);
2450                         }
2451                         break;
2452 
2453                 case ENOENT:
2454                         break;
2455 
2456                 case ECONNABORTED:
2457                         libscf_handle_rebind(info->h);
2458                         rebound = B_TRUE;
2459                         goto rebound;
2460 
2461                 case EINVAL:
2462                 case ENOTSUP:
2463                 default:
2464                         bad_error("libscf_fmri_get_instance", err);
2465                 }
2466 
2467                 scf_instance_destroy(inst);
2468 
2469                 if (rebound)
2470                         return (info->err = ECONNRESET);
2471                 break;
2472 
2473         case GVT_SVC:
2474                 (void) add_service(v->gv_name, info->h, &rebound);
2475                 if (rebound)
2476                         return (info->err = ECONNRESET);
2477         }
2478 
2479         return (0);
2480 }
2481 
2482 struct deppg_info {
2483         graph_vertex_t  *v;             /* GVT_INST vertex */
2484         int             err;            /* return error */
2485         int             **pathp;        /* return circular dependency path */
2486 };
2487 
2488 /*
2489  * Make info->v depend on a new GVT_GROUP node for this property group,
2490  * and then call process_dependency_fmri() for the values of the entity
2491  * property.  Return 0 on success, or if something goes wrong return nonzero
2492  * and set info->err to ECONNABORTED, EINVAL, or the error code returned by
2493  * process_dependency_fmri().
2494  */
2495 static int
2496 process_dependency_pg(scf_propertygroup_t *pg, struct deppg_info *info)
2497 {
2498         scf_handle_t *h;
2499         depgroup_type_t deptype;
2500         restarter_error_t rerr;
2501         struct depfmri_info linfo;
2502         char *fmri, *pg_name;
2503         size_t fmri_sz;
2504         graph_vertex_t *depgrp;
2505         scf_property_t *prop;
2506         int err;
2507         int empty;
2508         scf_error_t scferr;
2509         ssize_t len;
2510 
2511         assert(MUTEX_HELD(&dgraph_lock));
2512 
2513         h = scf_pg_handle(pg);
2514 
2515         pg_name = startd_alloc(max_scf_name_size);
2516 
2517         len = scf_pg_get_name(pg, pg_name, max_scf_name_size);
2518         if (len < 0) {
2519                 startd_free(pg_name, max_scf_name_size);
2520                 switch (scf_error()) {
2521                 case SCF_ERROR_CONNECTION_BROKEN:
2522                 default:
2523                         return (info->err = ECONNABORTED);
2524 
2525                 case SCF_ERROR_DELETED:
2526                         return (info->err = 0);
2527 
2528                 case SCF_ERROR_NOT_SET:
2529                         bad_error("scf_pg_get_name", scf_error());
2530                 }
2531         }
2532 
2533         /*
2534          * Skip over empty dependency groups.  Since dependency property
2535          * groups are updated atomically, they are either empty or
2536          * fully populated.
2537          */
2538         empty = depgroup_empty(h, pg);
2539         if (empty < 0) {
2540                 log_error(LOG_INFO,
2541                     "Error reading dependency group \"%s\" of %s: %s\n",
2542                     pg_name, info->v->gv_name, scf_strerror(scf_error()));
2543                 startd_free(pg_name, max_scf_name_size);
2544                 return (info->err = EINVAL);
2545 
2546         } else if (empty == 1) {
2547                 log_framework(LOG_DEBUG,
2548                     "Ignoring empty dependency group \"%s\" of %s\n",
2549                     pg_name, info->v->gv_name);
2550                 startd_free(pg_name, max_scf_name_size);
2551                 return (info->err = 0);
2552         }
2553 
2554         fmri_sz = strlen(info->v->gv_name) + 1 + len + 1;
2555         fmri = startd_alloc(fmri_sz);
2556 
2557         (void) snprintf(fmri, fmri_sz, "%s>%s", info->v->gv_name,
2558             pg_name);
2559 
2560         /* Validate the pg before modifying the graph */
2561         deptype = depgroup_read_grouping(h, pg);
2562         if (deptype == DEPGRP_UNSUPPORTED) {
2563                 log_error(LOG_INFO,
2564                     "Dependency \"%s\" of %s has an unknown grouping value.\n",
2565                     pg_name, info->v->gv_name);
2566                 startd_free(fmri, fmri_sz);
2567                 startd_free(pg_name, max_scf_name_size);
2568                 return (info->err = EINVAL);
2569         }
2570 
2571         rerr = depgroup_read_restart(h, pg);
2572         if (rerr == RERR_UNSUPPORTED) {
2573                 log_error(LOG_INFO,
2574                     "Dependency \"%s\" of %s has an unknown restart_on value."
2575                     "\n", pg_name, info->v->gv_name);
2576                 startd_free(fmri, fmri_sz);
2577                 startd_free(pg_name, max_scf_name_size);
2578                 return (info->err = EINVAL);
2579         }
2580 
2581         prop = safe_scf_property_create(h);
2582 
2583         if (scf_pg_get_property(pg, SCF_PROPERTY_ENTITIES, prop) != 0) {
2584                 scferr = scf_error();
2585                 scf_property_destroy(prop);
2586                 if (scferr == SCF_ERROR_DELETED) {
2587                         startd_free(fmri, fmri_sz);
2588                         startd_free(pg_name, max_scf_name_size);
2589                         return (info->err = 0);
2590                 } else if (scferr != SCF_ERROR_NOT_FOUND) {
2591                         startd_free(fmri, fmri_sz);
2592                         startd_free(pg_name, max_scf_name_size);
2593                         return (info->err = ECONNABORTED);
2594                 }
2595 
2596                 log_error(LOG_INFO,
2597                     "Dependency \"%s\" of %s is missing a \"%s\" property.\n",
2598                     pg_name, info->v->gv_name, SCF_PROPERTY_ENTITIES);
2599 
2600                 startd_free(fmri, fmri_sz);
2601                 startd_free(pg_name, max_scf_name_size);
2602 
2603                 return (info->err = EINVAL);
2604         }
2605 
2606         /* Create depgroup vertex for pg */
2607         err = graph_insert_vertex_unconfigured(fmri, GVT_GROUP, deptype,
2608             rerr, &depgrp);
2609         assert(err == 0);
2610         startd_free(fmri, fmri_sz);
2611 
2612         /* Add dependency from inst vertex to new vertex */
2613         err = graph_insert_dependency(info->v, depgrp, info->pathp);
2614         /* ELOOP can't happen because this should be a new vertex */
2615         assert(err == 0);
2616 
2617         linfo.v = depgrp;
2618         linfo.type = depgroup_read_scheme(h, pg);
2619         linfo.inst_fmri = info->v->gv_name;
2620         linfo.pg_name = pg_name;
2621         linfo.h = h;
2622         linfo.err = 0;
2623         linfo.pathp = info->pathp;
2624         err = walk_property_astrings(prop, (callback_t)process_dependency_fmri,
2625             &linfo);
2626 
2627         scf_property_destroy(prop);
2628         startd_free(pg_name, max_scf_name_size);
2629 
2630         switch (err) {
2631         case 0:
2632         case EINTR:
2633                 return (info->err = linfo.err);
2634 
2635         case ECONNABORTED:
2636         case EINVAL:
2637                 return (info->err = err);
2638 
2639         case ECANCELED:
2640                 return (info->err = 0);
2641 
2642         case ECONNRESET:
2643                 return (info->err = ECONNABORTED);
2644 
2645         default:
2646                 bad_error("walk_property_astrings", err);
2647                 /* NOTREACHED */
2648         }
2649 }
2650 
2651 /*
2652  * Build the dependency info for v from the repository.  Returns 0 on success,
2653  * ECONNABORTED on repository disconnection, EINVAL if the repository
2654  * configuration is invalid, and ELOOP if a dependency would cause a cycle.
2655  * In the last case, *pathp will point to a -1-terminated array of ids which
2656  * constitute the rest of the dependency cycle.
2657  */
2658 static int
2659 set_dependencies(graph_vertex_t *v, scf_instance_t *inst, int **pathp)
2660 {
2661         struct deppg_info info;
2662         int err;
2663         uint_t old_configured;
2664 
2665         assert(MUTEX_HELD(&dgraph_lock));
2666 
2667         /*
2668          * Mark the vertex as configured during dependency insertion to avoid
2669          * dependency cycles (which can appear in the graph if one of the
2670          * vertices is an exclusion-group).
2671          */
2672         old_configured = v->gv_flags & GV_CONFIGURED;
2673         v->gv_flags |= GV_CONFIGURED;
2674 
2675         info.err = 0;
2676         info.v = v;
2677         info.pathp = pathp;
2678 
2679         err = walk_dependency_pgs(inst, (callback_t)process_dependency_pg,
2680             &info);
2681 
2682         if (!old_configured)
2683                 v->gv_flags &= ~GV_CONFIGURED;
2684 
2685         switch (err) {
2686         case 0:
2687         case EINTR:
2688                 return (info.err);
2689 
2690         case ECONNABORTED:
2691                 return (ECONNABORTED);
2692 
2693         case ECANCELED:
2694                 /* Should get delete event, so return 0. */
2695                 return (0);
2696 
2697         default:
2698                 bad_error("walk_dependency_pgs", err);
2699                 /* NOTREACHED */
2700         }
2701 }
2702 
2703 
2704 static void
2705 handle_cycle(const char *fmri, int *path)
2706 {
2707         const char *cp;
2708         size_t sz;
2709 
2710         assert(MUTEX_HELD(&dgraph_lock));
2711 
2712         path_to_str(path, (char **)&cp, &sz);
2713 
2714         log_error(LOG_ERR, "Transitioning %s to maintenance "
2715             "because it completes a dependency cycle (see svcs -xv for "
2716             "details):\n%s", fmri ? fmri : "?", cp);
2717 
2718         startd_free((void *)cp, sz);
2719 }
2720 
2721 /*
2722  * Increment the vertex's reference count to prevent the vertex removal
2723  * from the dgraph.
2724  */
2725 static void
2726 vertex_ref(graph_vertex_t *v)
2727 {
2728         assert(MUTEX_HELD(&dgraph_lock));
2729 
2730         v->gv_refs++;
2731 }
2732 
2733 /*
2734  * Decrement the vertex's reference count and remove the vertex from
2735  * the dgraph when possible.
2736  *
2737  * Return VERTEX_REMOVED when the vertex has been removed otherwise
2738  * return VERTEX_INUSE.
2739  */
2740 static int
2741 vertex_unref(graph_vertex_t *v)
2742 {
2743         assert(MUTEX_HELD(&dgraph_lock));
2744         assert(v->gv_refs > 0);
2745 
2746         v->gv_refs--;
2747 
2748         return (free_if_unrefed(v));
2749 }
2750 
2751 /*
2752  * When run on the dependencies of a vertex, populates list with
2753  * graph_edge_t's which point to the service vertices or the instance
2754  * vertices (no GVT_GROUP nodes) on which the vertex depends.
2755  *
2756  * Increment the vertex's reference count once the vertex is inserted
2757  * in the list. The vertex won't be able to be deleted from the dgraph
2758  * while it is referenced.
2759  */
2760 static int
2761 append_svcs_or_insts(graph_edge_t *e, uu_list_t *list)
2762 {
2763         graph_vertex_t *v = e->ge_vertex;
2764         graph_edge_t *new;
2765         int r;
2766 
2767         switch (v->gv_type) {
2768         case GVT_INST:
2769         case GVT_SVC:
2770                 break;
2771 
2772         case GVT_GROUP:
2773                 r = uu_list_walk(v->gv_dependencies,
2774                     (uu_walk_fn_t *)append_svcs_or_insts, list, 0);
2775                 assert(r == 0);
2776                 return (UU_WALK_NEXT);
2777 
2778         case GVT_FILE:
2779                 return (UU_WALK_NEXT);
2780 
2781         default:
2782 #ifndef NDEBUG
2783                 uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
2784                     __LINE__, v->gv_type);
2785 #endif
2786                 abort();
2787         }
2788 
2789         new = startd_alloc(sizeof (*new));
2790         new->ge_vertex = v;
2791         uu_list_node_init(new, &new->ge_link, graph_edge_pool);
2792         r = uu_list_insert_before(list, NULL, new);
2793         assert(r == 0);
2794 
2795         /*
2796          * Because we are inserting the vertex in a list, we don't want
2797          * the vertex to be freed while the list is in use. In order to
2798          * achieve that, increment the vertex's reference count.
2799          */
2800         vertex_ref(v);
2801 
2802         return (UU_WALK_NEXT);
2803 }
2804 
2805 static boolean_t
2806 should_be_in_subgraph(graph_vertex_t *v)
2807 {
2808         graph_edge_t *e;
2809 
2810         if (v == milestone)
2811                 return (B_TRUE);
2812 
2813         /*
2814          * v is in the subgraph if any of its dependents are in the subgraph.
2815          * Except for EXCLUDE_ALL dependents.  And OPTIONAL dependents only
2816          * count if we're enabled.
2817          */
2818         for (e = uu_list_first(v->gv_dependents);
2819             e != NULL;
2820             e = uu_list_next(v->gv_dependents, e)) {
2821                 graph_vertex_t *dv = e->ge_vertex;
2822 
2823                 if (!(dv->gv_flags & GV_INSUBGRAPH))
2824                         continue;
2825 
2826                 /*
2827                  * Don't include instances that are optional and disabled.
2828                  */
2829                 if (v->gv_type == GVT_INST && dv->gv_type == GVT_SVC) {
2830 
2831                         int in = 0;
2832                         graph_edge_t *ee;
2833 
2834                         for (ee = uu_list_first(dv->gv_dependents);
2835                             ee != NULL;
2836                             ee = uu_list_next(dv->gv_dependents, ee)) {
2837 
2838                                 graph_vertex_t *ddv = e->ge_vertex;
2839 
2840                                 if (ddv->gv_type == GVT_GROUP &&
2841                                     ddv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2842                                         continue;
2843 
2844                                 if (ddv->gv_type == GVT_GROUP &&
2845                                     ddv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2846                                     !(v->gv_flags & GV_ENBLD_NOOVR))
2847                                         continue;
2848 
2849                                 in = 1;
2850                         }
2851                         if (!in)
2852                                 continue;
2853                 }
2854                 if (v->gv_type == GVT_INST &&
2855                     dv->gv_type == GVT_GROUP &&
2856                     dv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2857                     !(v->gv_flags & GV_ENBLD_NOOVR))
2858                         continue;
2859 
2860                 /* Don't include excluded services and instances */
2861                 if (dv->gv_type == GVT_GROUP &&
2862                     dv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2863                         continue;
2864 
2865                 return (B_TRUE);
2866         }
2867 
2868         return (B_FALSE);
2869 }
2870 
2871 /*
2872  * Ensures that GV_INSUBGRAPH is set properly for v and its descendents.  If
2873  * any bits change, manipulate the repository appropriately.  Returns 0 or
2874  * ECONNABORTED.
2875  */
2876 static int
2877 eval_subgraph(graph_vertex_t *v, scf_handle_t *h)
2878 {
2879         boolean_t old = (v->gv_flags & GV_INSUBGRAPH) != 0;
2880         boolean_t new;
2881         graph_edge_t *e;
2882         scf_instance_t *inst;
2883         int ret = 0, r;
2884 
2885         assert(milestone != NULL && milestone != MILESTONE_NONE);
2886 
2887         new = should_be_in_subgraph(v);
2888 
2889         if (new == old)
2890                 return (0);
2891 
2892         log_framework(LOG_DEBUG, new ? "Adding %s to the subgraph.\n" :
2893             "Removing %s from the subgraph.\n", v->gv_name);
2894 
2895         v->gv_flags = (v->gv_flags & ~GV_INSUBGRAPH) |
2896             (new ? GV_INSUBGRAPH : 0);
2897 
2898         if (v->gv_type == GVT_INST && (v->gv_flags & GV_CONFIGURED)) {
2899                 int err;
2900 
2901 get_inst:
2902                 err = libscf_fmri_get_instance(h, v->gv_name, &inst);
2903                 if (err != 0) {
2904                         switch (err) {
2905                         case ECONNABORTED:
2906                                 libscf_handle_rebind(h);
2907                                 ret = ECONNABORTED;
2908                                 goto get_inst;
2909 
2910                         case ENOENT:
2911                                 break;
2912 
2913                         case EINVAL:
2914                         case ENOTSUP:
2915                         default:
2916                                 bad_error("libscf_fmri_get_instance", err);
2917                         }
2918                 } else {
2919                         const char *f;
2920 
2921                         if (new) {
2922                                 err = libscf_delete_enable_ovr(inst);
2923                                 f = "libscf_delete_enable_ovr";
2924                         } else {
2925                                 err = libscf_set_enable_ovr(inst, 0);
2926                                 f = "libscf_set_enable_ovr";
2927                         }
2928                         scf_instance_destroy(inst);
2929                         switch (err) {
2930                         case 0:
2931                         case ECANCELED:
2932                                 break;
2933 
2934                         case ECONNABORTED:
2935                                 libscf_handle_rebind(h);
2936                                 /*
2937                                  * We must continue so the graph is updated,
2938                                  * but we must return ECONNABORTED so any
2939                                  * libscf state held by any callers is reset.
2940                                  */
2941                                 ret = ECONNABORTED;
2942                                 goto get_inst;
2943 
2944                         case EROFS:
2945                         case EPERM:
2946                                 log_error(LOG_WARNING,
2947                                     "Could not set %s/%s for %s: %s.\n",
2948                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
2949                                     v->gv_name, strerror(err));
2950                                 break;
2951 
2952                         default:
2953                                 bad_error(f, err);
2954                         }
2955                 }
2956         }
2957 
2958         for (e = uu_list_first(v->gv_dependencies);
2959             e != NULL;
2960             e = uu_list_next(v->gv_dependencies, e)) {
2961                 r = eval_subgraph(e->ge_vertex, h);
2962                 if (r != 0) {
2963                         assert(r == ECONNABORTED);
2964                         ret = ECONNABORTED;
2965                 }
2966         }
2967 
2968         return (ret);
2969 }
2970 
2971 /*
2972  * Delete the (property group) dependencies of v & create new ones based on
2973  * inst.  If doing so would create a cycle, log a message and put the instance
2974  * into maintenance.  Update GV_INSUBGRAPH flags as necessary.  Returns 0 or
2975  * ECONNABORTED.
2976  */
2977 int
2978 refresh_vertex(graph_vertex_t *v, scf_instance_t *inst)
2979 {
2980         int err;
2981         int *path;
2982         char *fmri;
2983         int r;
2984         scf_handle_t *h = scf_instance_handle(inst);
2985         uu_list_t *old_deps;
2986         int ret = 0;
2987         graph_edge_t *e;
2988         graph_vertex_t *vv;
2989 
2990         assert(MUTEX_HELD(&dgraph_lock));
2991         assert(v->gv_type == GVT_INST);
2992 
2993         log_framework(LOG_DEBUG, "Graph engine: Refreshing %s.\n", v->gv_name);
2994 
2995         if (milestone > MILESTONE_NONE) {
2996                 /*
2997                  * In case some of v's dependencies are being deleted we must
2998                  * make a list of them now for GV_INSUBGRAPH-flag evaluation
2999                  * after the new dependencies are in place.
3000                  */
3001                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
3002 
3003                 err = uu_list_walk(v->gv_dependencies,
3004                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
3005                 assert(err == 0);
3006         }
3007 
3008         delete_instance_dependencies(v, B_FALSE);
3009 
3010         err = set_dependencies(v, inst, &path);
3011         switch (err) {
3012         case 0:
3013                 break;
3014 
3015         case ECONNABORTED:
3016                 ret = err;
3017                 goto out;
3018 
3019         case EINVAL:
3020         case ELOOP:
3021                 r = libscf_instance_get_fmri(inst, &fmri);
3022                 switch (r) {
3023                 case 0:
3024                         break;
3025 
3026                 case ECONNABORTED:
3027                         ret = ECONNABORTED;
3028                         goto out;
3029 
3030                 case ECANCELED:
3031                         ret = 0;
3032                         goto out;
3033 
3034                 default:
3035                         bad_error("libscf_instance_get_fmri", r);
3036                 }
3037 
3038                 if (err == EINVAL) {
3039                         log_error(LOG_ERR, "Transitioning %s "
3040                             "to maintenance due to misconfiguration.\n",
3041                             fmri ? fmri : "?");
3042                         vertex_send_event(v,
3043                             RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY);
3044                 } else {
3045                         handle_cycle(fmri, path);
3046                         vertex_send_event(v,
3047                             RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE);
3048                 }
3049                 startd_free(fmri, max_scf_fmri_size);
3050                 ret = 0;
3051                 goto out;
3052 
3053         default:
3054                 bad_error("set_dependencies", err);
3055         }
3056 
3057         if (milestone > MILESTONE_NONE) {
3058                 boolean_t aborted = B_FALSE;
3059 
3060                 for (e = uu_list_first(old_deps);
3061                     e != NULL;
3062                     e = uu_list_next(old_deps, e)) {
3063                         vv = e->ge_vertex;
3064 
3065                         if (vertex_unref(vv) == VERTEX_INUSE &&
3066                             eval_subgraph(vv, h) == ECONNABORTED)
3067                                 aborted = B_TRUE;
3068                 }
3069 
3070                 for (e = uu_list_first(v->gv_dependencies);
3071                     e != NULL;
3072                     e = uu_list_next(v->gv_dependencies, e)) {
3073                         if (eval_subgraph(e->ge_vertex, h) ==
3074                             ECONNABORTED)
3075                                 aborted = B_TRUE;
3076                 }
3077 
3078                 if (aborted) {
3079                         ret = ECONNABORTED;
3080                         goto out;
3081                 }
3082         }
3083 
3084         graph_start_if_satisfied(v);
3085 
3086         ret = 0;
3087 
3088 out:
3089         if (milestone > MILESTONE_NONE) {
3090                 void *cookie = NULL;
3091 
3092                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL)
3093                         startd_free(e, sizeof (*e));
3094 
3095                 uu_list_destroy(old_deps);
3096         }
3097 
3098         return (ret);
3099 }
3100 
3101 /*
3102  * Set up v according to inst.  That is, make sure it depends on its
3103  * restarter and set up its dependencies.  Send the ADD_INSTANCE command to
3104  * the restarter, and send ENABLE or DISABLE as appropriate.
3105  *
3106  * Returns 0 on success, ECONNABORTED on repository disconnection, or
3107  * ECANCELED if inst is deleted.
3108  */
3109 static int
3110 configure_vertex(graph_vertex_t *v, scf_instance_t *inst)
3111 {
3112         scf_handle_t *h;
3113         scf_propertygroup_t *pg;
3114         scf_snapshot_t *snap;
3115         char *restarter_fmri = startd_alloc(max_scf_value_size);
3116         int enabled, enabled_ovr;
3117         int err;
3118         int *path;
3119         int deathrow;
3120         int32_t tset;
3121 
3122         restarter_fmri[0] = '\0';
3123 
3124         assert(MUTEX_HELD(&dgraph_lock));
3125         assert(v->gv_type == GVT_INST);
3126         assert((v->gv_flags & GV_CONFIGURED) == 0);
3127 
3128         /* GV_INSUBGRAPH should already be set properly. */
3129         assert(should_be_in_subgraph(v) ==
3130             ((v->gv_flags & GV_INSUBGRAPH) != 0));
3131 
3132         /*
3133          * If the instance fmri is in the deathrow list then set the
3134          * GV_DEATHROW flag on the vertex and create and set to true the
3135          * SCF_PROPERTY_DEATHROW boolean property in the non-persistent
3136          * repository for this instance fmri.
3137          */
3138         if ((v->gv_flags & GV_DEATHROW) ||
3139             (is_fmri_in_deathrow(v->gv_name) == B_TRUE)) {
3140                 if ((v->gv_flags & GV_DEATHROW) == 0) {
3141                         /*
3142                          * Set flag GV_DEATHROW, create and set to true
3143                          * the SCF_PROPERTY_DEATHROW property in the
3144                          * non-persistent repository for this instance fmri.
3145                          */
3146                         v->gv_flags |= GV_DEATHROW;
3147 
3148                         switch (err = libscf_set_deathrow(inst, 1)) {
3149                         case 0:
3150                                 break;
3151 
3152                         case ECONNABORTED:
3153                         case ECANCELED:
3154                                 startd_free(restarter_fmri, max_scf_value_size);
3155                                 return (err);
3156 
3157                         case EROFS:
3158                                 log_error(LOG_WARNING, "Could not set %s/%s "
3159                                     "for deathrow %s: %s.\n",
3160                                     SCF_PG_DEATHROW, SCF_PROPERTY_DEATHROW,
3161                                     v->gv_name, strerror(err));
3162                                 break;
3163 
3164                         case EPERM:
3165                                 uu_die("Permission denied.\n");
3166                                 /* NOTREACHED */
3167 
3168                         default:
3169                                 bad_error("libscf_set_deathrow", err);
3170                         }
3171                         log_framework(LOG_DEBUG, "Deathrow, graph set %s.\n",
3172                             v->gv_name);
3173                 }
3174                 startd_free(restarter_fmri, max_scf_value_size);
3175                 return (0);
3176         }
3177 
3178         h = scf_instance_handle(inst);
3179 
3180         /*
3181          * Using a temporary deathrow boolean property, set through
3182          * libscf_set_deathrow(), only for fmris on deathrow, is necessary
3183          * because deathrow_fini() may already have been called, and in case
3184          * of a refresh, GV_DEATHROW may need to be set again.
3185          * libscf_get_deathrow() sets deathrow to 1 only if this instance
3186          * has a temporary boolean property named 'deathrow' valued true
3187          * in a property group 'deathrow', -1 or 0 in all other cases.
3188          */
3189         err = libscf_get_deathrow(h, inst, &deathrow);
3190         switch (err) {
3191         case 0:
3192                 break;
3193 
3194         case ECONNABORTED:
3195         case ECANCELED:
3196                 startd_free(restarter_fmri, max_scf_value_size);
3197                 return (err);
3198 
3199         default:
3200                 bad_error("libscf_get_deathrow", err);
3201         }
3202 
3203         if (deathrow == 1) {
3204                 v->gv_flags |= GV_DEATHROW;
3205                 startd_free(restarter_fmri, max_scf_value_size);
3206                 return (0);
3207         }
3208 
3209         log_framework(LOG_DEBUG, "Graph adding %s.\n", v->gv_name);
3210 
3211         /*
3212          * If the instance does not have a restarter property group,
3213          * initialize its state to uninitialized/none, in case the restarter
3214          * is not enabled.
3215          */
3216         pg = safe_scf_pg_create(h);
3217 
3218         if (scf_instance_get_pg(inst, SCF_PG_RESTARTER, pg) != 0) {
3219                 instance_data_t idata;
3220                 uint_t count = 0, msecs = ALLOC_DELAY;
3221 
3222                 switch (scf_error()) {
3223                 case SCF_ERROR_NOT_FOUND:
3224                         break;
3225 
3226                 case SCF_ERROR_CONNECTION_BROKEN:
3227                 default:
3228                         scf_pg_destroy(pg);
3229                         startd_free(restarter_fmri, max_scf_value_size);
3230                         return (ECONNABORTED);
3231 
3232                 case SCF_ERROR_DELETED:
3233                         scf_pg_destroy(pg);
3234                         startd_free(restarter_fmri, max_scf_value_size);
3235                         return (ECANCELED);
3236 
3237                 case SCF_ERROR_NOT_SET:
3238                         bad_error("scf_instance_get_pg", scf_error());
3239                 }
3240 
3241                 switch (err = libscf_instance_get_fmri(inst,
3242                     (char **)&idata.i_fmri)) {
3243                 case 0:
3244                         break;
3245 
3246                 case ECONNABORTED:
3247                 case ECANCELED:
3248                         scf_pg_destroy(pg);
3249                         startd_free(restarter_fmri, max_scf_value_size);
3250                         return (err);
3251 
3252                 default:
3253                         bad_error("libscf_instance_get_fmri", err);
3254                 }
3255 
3256                 idata.i_state = RESTARTER_STATE_NONE;
3257                 idata.i_next_state = RESTARTER_STATE_NONE;
3258 
3259 init_state:
3260                 switch (err = _restarter_commit_states(h, &idata,
3261                     RESTARTER_STATE_UNINIT, RESTARTER_STATE_NONE,
3262                     restarter_get_str_short(restarter_str_insert_in_graph))) {
3263                 case 0:
3264                         break;
3265 
3266                 case ENOMEM:
3267                         ++count;
3268                         if (count < ALLOC_RETRY) {
3269                                 (void) poll(NULL, 0, msecs);
3270                                 msecs *= ALLOC_DELAY_MULT;
3271                                 goto init_state;
3272                         }
3273 
3274                         uu_die("Insufficient memory.\n");
3275                         /* NOTREACHED */
3276 
3277                 case ECONNABORTED:
3278                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3279                         scf_pg_destroy(pg);
3280                         startd_free(restarter_fmri, max_scf_value_size);
3281                         return (ECONNABORTED);
3282 
3283                 case ENOENT:
3284                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3285                         scf_pg_destroy(pg);
3286                         startd_free(restarter_fmri, max_scf_value_size);
3287                         return (ECANCELED);
3288 
3289                 case EPERM:
3290                 case EACCES:
3291                 case EROFS:
3292                         log_error(LOG_NOTICE, "Could not initialize state for "
3293                             "%s: %s.\n", idata.i_fmri, strerror(err));
3294                         break;
3295 
3296                 case EINVAL:
3297                 default:
3298                         bad_error("_restarter_commit_states", err);
3299                 }
3300 
3301                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3302         }
3303 
3304         scf_pg_destroy(pg);
3305 
3306         if (milestone != NULL) {
3307                 /*
3308                  * Make sure the enable-override is set properly before we
3309                  * read whether we should be enabled.
3310                  */
3311                 if (milestone == MILESTONE_NONE ||
3312                     !(v->gv_flags & GV_INSUBGRAPH)) {
3313                         /*
3314                          * This might seem unjustified after the milestone
3315                          * transition has completed (non_subgraph_svcs == 0),
3316                          * but it's important because when we boot to
3317                          * a milestone, we set the milestone before populating
3318                          * the graph, and all of the new non-subgraph services
3319                          * need to be disabled here.
3320                          */
3321                         switch (err = libscf_set_enable_ovr(inst, 0)) {
3322                         case 0:
3323                                 break;
3324 
3325                         case ECONNABORTED:
3326                         case ECANCELED:
3327                                 startd_free(restarter_fmri, max_scf_value_size);
3328                                 return (err);
3329 
3330                         case EROFS:
3331                                 log_error(LOG_WARNING,
3332                                     "Could not set %s/%s for %s: %s.\n",
3333                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
3334                                     v->gv_name, strerror(err));
3335                                 break;
3336 
3337                         case EPERM:
3338                                 uu_die("Permission denied.\n");
3339                                 /* NOTREACHED */
3340 
3341                         default:
3342                                 bad_error("libscf_set_enable_ovr", err);
3343                         }
3344                 } else {
3345                         assert(v->gv_flags & GV_INSUBGRAPH);
3346                         switch (err = libscf_delete_enable_ovr(inst)) {
3347                         case 0:
3348                                 break;
3349 
3350                         case ECONNABORTED:
3351                         case ECANCELED:
3352                                 startd_free(restarter_fmri, max_scf_value_size);
3353                                 return (err);
3354 
3355                         case EPERM:
3356                                 uu_die("Permission denied.\n");
3357                                 /* NOTREACHED */
3358 
3359                         default:
3360                                 bad_error("libscf_delete_enable_ovr", err);
3361                         }
3362                 }
3363         }
3364 
3365         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
3366             &enabled_ovr, &restarter_fmri);
3367         switch (err) {
3368         case 0:
3369                 break;
3370 
3371         case ECONNABORTED:
3372         case ECANCELED:
3373                 startd_free(restarter_fmri, max_scf_value_size);
3374                 return (err);
3375 
3376         case ENOENT:
3377                 log_framework(LOG_DEBUG,
3378                     "Ignoring %s because it has no general property group.\n",
3379                     v->gv_name);
3380                 startd_free(restarter_fmri, max_scf_value_size);
3381                 return (0);
3382 
3383         default:
3384                 bad_error("libscf_get_basic_instance_data", err);
3385         }
3386 
3387         if ((tset = libscf_get_stn_tset(inst)) == -1) {
3388                 log_framework(LOG_WARNING,
3389                     "Failed to get notification parameters for %s: %s\n",
3390                     v->gv_name, scf_strerror(scf_error()));
3391                 v->gv_stn_tset = 0;
3392         } else {
3393                 v->gv_stn_tset = tset;
3394         }
3395         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
3396                 stn_global = v->gv_stn_tset;
3397 
3398         if (enabled == -1) {
3399                 startd_free(restarter_fmri, max_scf_value_size);
3400                 return (0);
3401         }
3402 
3403         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
3404             (enabled ? GV_ENBLD_NOOVR : 0);
3405 
3406         if (enabled_ovr != -1)
3407                 enabled = enabled_ovr;
3408 
3409         v->gv_state = RESTARTER_STATE_UNINIT;
3410 
3411         snap = libscf_get_or_make_running_snapshot(inst, v->gv_name, B_TRUE);
3412         scf_snapshot_destroy(snap);
3413 
3414         /* Set up the restarter. (Sends _ADD_INSTANCE on success.) */
3415         err = graph_change_restarter(v, restarter_fmri, h, &path);
3416         if (err != 0) {
3417                 instance_data_t idata;
3418                 uint_t count = 0, msecs = ALLOC_DELAY;
3419                 restarter_str_t reason;
3420 
3421                 if (err == ECONNABORTED) {
3422                         startd_free(restarter_fmri, max_scf_value_size);
3423                         return (err);
3424                 }
3425 
3426                 assert(err == EINVAL || err == ELOOP);
3427 
3428                 if (err == EINVAL) {
3429                         log_framework(LOG_ERR, emsg_invalid_restarter,
3430                             v->gv_name, restarter_fmri);
3431                         reason = restarter_str_invalid_restarter;
3432                 } else {
3433                         handle_cycle(v->gv_name, path);
3434                         reason = restarter_str_dependency_cycle;
3435                 }
3436 
3437                 startd_free(restarter_fmri, max_scf_value_size);
3438 
3439                 /*
3440                  * We didn't register the instance with the restarter, so we
3441                  * must set maintenance mode ourselves.
3442                  */
3443                 err = libscf_instance_get_fmri(inst, (char **)&idata.i_fmri);
3444                 if (err != 0) {
3445                         assert(err == ECONNABORTED || err == ECANCELED);
3446                         return (err);
3447                 }
3448 
3449                 idata.i_state = RESTARTER_STATE_NONE;
3450                 idata.i_next_state = RESTARTER_STATE_NONE;
3451 
3452 set_maint:
3453                 switch (err = _restarter_commit_states(h, &idata,
3454                     RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE,
3455                     restarter_get_str_short(reason))) {
3456                 case 0:
3457                         break;
3458 
3459                 case ENOMEM:
3460                         ++count;
3461                         if (count < ALLOC_RETRY) {
3462                                 (void) poll(NULL, 0, msecs);
3463                                 msecs *= ALLOC_DELAY_MULT;
3464                                 goto set_maint;
3465                         }
3466 
3467                         uu_die("Insufficient memory.\n");
3468                         /* NOTREACHED */
3469 
3470                 case ECONNABORTED:
3471                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3472                         return (ECONNABORTED);
3473 
3474                 case ENOENT:
3475                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3476                         return (ECANCELED);
3477 
3478                 case EPERM:
3479                 case EACCES:
3480                 case EROFS:
3481                         log_error(LOG_NOTICE, "Could not initialize state for "
3482                             "%s: %s.\n", idata.i_fmri, strerror(err));
3483                         break;
3484 
3485                 case EINVAL:
3486                 default:
3487                         bad_error("_restarter_commit_states", err);
3488                 }
3489 
3490                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3491 
3492                 v->gv_state = RESTARTER_STATE_MAINT;
3493 
3494                 goto out;
3495         }
3496         startd_free(restarter_fmri, max_scf_value_size);
3497 
3498         /* Add all the other dependencies. */
3499         err = refresh_vertex(v, inst);
3500         if (err != 0) {
3501                 assert(err == ECONNABORTED);
3502                 return (err);
3503         }
3504 
3505 out:
3506         v->gv_flags |= GV_CONFIGURED;
3507 
3508         graph_enable_by_vertex(v, enabled, 0);
3509 
3510         return (0);
3511 }
3512 
3513 
3514 static void
3515 kill_user_procs(void)
3516 {
3517         (void) fputs("svc.startd: Killing user processes.\n", stdout);
3518 
3519         /*
3520          * Despite its name, killall's role is to get select user processes--
3521          * basically those representing terminal-based logins-- to die.  Victims
3522          * are located by killall in the utmp database.  Since these are most
3523          * often shell based logins, and many shells mask SIGTERM (but are
3524          * responsive to SIGHUP) we first HUP and then shortly thereafter
3525          * kill -9.
3526          */
3527         (void) fork_with_timeout("/usr/sbin/killall HUP", 1, 5);
3528         (void) fork_with_timeout("/usr/sbin/killall KILL", 1, 5);
3529 
3530         /*
3531          * Note the selection of user id's 0, 1 and 15, subsequently
3532          * inverted by -v.  15 is reserved for dladmd.  Yes, this is a
3533          * kludge-- a better policy is needed.
3534          *
3535          * Note that fork_with_timeout will only wait out the 1 second
3536          * "grace time" if pkill actually returns 0.  So if there are
3537          * no matches, this will run to completion much more quickly.
3538          */
3539         (void) fork_with_timeout("/usr/bin/pkill -TERM -v -u 0,1,15", 1, 5);
3540         (void) fork_with_timeout("/usr/bin/pkill -KILL -v -u 0,1,15", 1, 5);
3541 }
3542 
3543 static void
3544 do_uadmin(void)
3545 {
3546         const char * const resetting = "/etc/svc/volatile/resetting";
3547         int fd;
3548         struct statvfs vfs;
3549         time_t now;
3550         struct tm nowtm;
3551         char down_buf[256], time_buf[256];
3552         uintptr_t mdep;
3553 #if defined(__i386)
3554         grub_boot_args_t fbarg;
3555 #endif  /* __i386 */
3556 
3557         mdep = NULL;
3558         fd = creat(resetting, 0777);
3559         if (fd >= 0)
3560                 startd_close(fd);
3561         else
3562                 uu_warn("Could not create \"%s\"", resetting);
3563 
3564         /* Kill dhcpagent if we're not using nfs for root */
3565         if ((statvfs("/", &vfs) == 0) &&
3566             (strncmp(vfs.f_basetype, "nfs", sizeof ("nfs") - 1) != 0))
3567                 fork_with_timeout("/usr/bin/pkill -x -u 0 dhcpagent", 0, 5);
3568 
3569         /*
3570          * Call sync(2) now, before we kill off user processes.  This takes
3571          * advantage of the several seconds of pause we have before the
3572          * killalls are done.  Time we can make good use of to get pages
3573          * moving out to disk.
3574          *
3575          * Inside non-global zones, we don't bother, and it's better not to
3576          * anyway, since sync(2) can have system-wide impact.
3577          */
3578         if (getzoneid() == 0)
3579                 sync();
3580 
3581         kill_user_procs();
3582 
3583         /*
3584          * Note that this must come after the killing of user procs, since
3585          * killall relies on utmpx, and this command affects the contents of
3586          * said file.
3587          */
3588         if (access("/usr/lib/acct/closewtmp", X_OK) == 0)
3589                 fork_with_timeout("/usr/lib/acct/closewtmp", 0, 5);
3590 
3591         /*
3592          * For patches which may be installed as the system is shutting
3593          * down, we need to ensure, one more time, that the boot archive
3594          * really is up to date.
3595          */
3596         if (getzoneid() == 0 && access("/usr/sbin/bootadm", X_OK) == 0)
3597                 fork_with_timeout("/usr/sbin/bootadm -ea update_all", 0, 3600);
3598 
3599         /*
3600          * Right now, fast reboot is supported only on i386.
3601          * scf_is_fastboot_default() should take care of it.
3602          * If somehow we got there on unsupported platform -
3603          * print warning and fall back to regular reboot.
3604          */
3605         if (halting == AD_FASTREBOOT) {
3606 #if defined(__i386)
3607                 int rc;
3608 
3609                 if ((rc = grub_get_boot_args(&fbarg, NULL,
3610                     GRUB_ENTRY_DEFAULT)) == 0) {
3611                         mdep = (uintptr_t)&fbarg.gba_bootargs;
3612                 } else {
3613                         /*
3614                          * Failed to read GRUB menu, fall back to normal reboot
3615                          */
3616                         halting = AD_BOOT;
3617                         uu_warn("Failed to process GRUB menu entry "
3618                             "for fast reboot.\n\t%s\n"
3619                             "Falling back to regular reboot.\n",
3620                             grub_strerror(rc));
3621                 }
3622 #else   /* __i386 */
3623                 halting = AD_BOOT;
3624                 uu_warn("Fast reboot configured, but not supported by "
3625                     "this ISA\n");
3626 #endif  /* __i386 */
3627         }
3628 
3629         fork_with_timeout("/sbin/umountall -l", 0, 5);
3630         fork_with_timeout("/sbin/umount /tmp /var/adm /var/run /var "
3631             ">/dev/null 2>&1", 0, 5);
3632 
3633         /*
3634          * Try to get to consistency for whatever UFS filesystems are left.
3635          * This is pretty expensive, so we save it for the end in the hopes of
3636          * minimizing what it must do.  The other option would be to start in
3637          * parallel with the killall's, but lockfs tends to throw out much more
3638          * than is needed, and so subsequent commands (like umountall) take a
3639          * long time to get going again.
3640          *
3641          * Inside of zones, we don't bother, since we're not about to terminate
3642          * the whole OS instance.
3643          *
3644          * On systems using only ZFS, this call to lockfs -fa is a no-op.
3645          */
3646         if (getzoneid() == 0) {
3647                 if (access("/usr/sbin/lockfs", X_OK) == 0)
3648                         fork_with_timeout("/usr/sbin/lockfs -fa", 0, 30);
3649 
3650                 sync(); /* once more, with feeling */
3651         }
3652 
3653         fork_with_timeout("/sbin/umount /usr >/dev/null 2>&1", 0, 5);
3654 
3655         /*
3656          * Construct and emit the last words from userland:
3657          * "<timestamp> The system is down.  Shutdown took <N> seconds."
3658          *
3659          * Normally we'd use syslog, but with /var and other things
3660          * potentially gone, try to minimize the external dependencies.
3661          */
3662         now = time(NULL);
3663         (void) localtime_r(&now, &nowtm);
3664 
3665         if (strftime(down_buf, sizeof (down_buf),
3666             "%b %e %T The system is down.", &nowtm) == 0) {
3667                 (void) strlcpy(down_buf, "The system is down.",
3668                     sizeof (down_buf));
3669         }
3670 
3671         if (halting_time != 0 && halting_time <= now) {
3672                 (void) snprintf(time_buf, sizeof (time_buf),
3673                     "  Shutdown took %lu seconds.", now - halting_time);
3674         } else {
3675                 time_buf[0] = '\0';
3676         }
3677         (void) printf("%s%s\n", down_buf, time_buf);
3678 
3679         (void) uadmin(A_SHUTDOWN, halting, mdep);
3680         uu_warn("uadmin() failed");
3681 
3682 #if defined(__i386)
3683         /* uadmin fail, cleanup grub_boot_args */
3684         if (halting == AD_FASTREBOOT)
3685                 grub_cleanup_boot_args(&fbarg);
3686 #endif  /* __i386 */
3687 
3688         if (remove(resetting) != 0 && errno != ENOENT)
3689                 uu_warn("Could not remove \"%s\"", resetting);
3690 }
3691 
3692 /*
3693  * If any of the up_svcs[] are online or satisfiable, return true.  If they are
3694  * all missing, disabled, in maintenance, or unsatisfiable, return false.
3695  */
3696 boolean_t
3697 can_come_up(void)
3698 {
3699         int i;
3700 
3701         assert(MUTEX_HELD(&dgraph_lock));
3702 
3703         /*
3704          * If we are booting to single user (boot -s),
3705          * SCF_MILESTONE_SINGLE_USER is needed to come up because startd
3706          * spawns sulogin after single-user is online (see specials.c).
3707          */
3708         i = (booting_to_single_user ? 0 : 1);
3709 
3710         for (; up_svcs[i] != NULL; ++i) {
3711                 if (up_svcs_p[i] == NULL) {
3712                         up_svcs_p[i] = vertex_get_by_name(up_svcs[i]);
3713 
3714                         if (up_svcs_p[i] == NULL)
3715                                 continue;
3716                 }
3717 
3718                 /*
3719                  * Ignore unconfigured services (the ones that have been
3720                  * mentioned in a dependency from other services, but do
3721                  * not exist in the repository).  Services which exist
3722                  * in the repository but don't have general/enabled
3723                  * property will be also ignored.
3724                  */
3725                 if (!(up_svcs_p[i]->gv_flags & GV_CONFIGURED))
3726                         continue;
3727 
3728                 switch (up_svcs_p[i]->gv_state) {
3729                 case RESTARTER_STATE_ONLINE:
3730                 case RESTARTER_STATE_DEGRADED:
3731                         /*
3732                          * Deactivate verbose boot once a login service has been
3733                          * reached.
3734                          */
3735                         st->st_log_login_reached = 1;
3736                         /*FALLTHROUGH*/
3737                 case RESTARTER_STATE_UNINIT:
3738                         return (B_TRUE);
3739 
3740                 case RESTARTER_STATE_OFFLINE:
3741                         if (instance_satisfied(up_svcs_p[i], B_TRUE) != -1)
3742                                 return (B_TRUE);
3743                         log_framework(LOG_DEBUG,
3744                             "can_come_up(): %s is unsatisfiable.\n",
3745                             up_svcs_p[i]->gv_name);
3746                         continue;
3747 
3748                 case RESTARTER_STATE_DISABLED:
3749                 case RESTARTER_STATE_MAINT:
3750                         log_framework(LOG_DEBUG,
3751                             "can_come_up(): %s is in state %s.\n",
3752                             up_svcs_p[i]->gv_name,
3753                             instance_state_str[up_svcs_p[i]->gv_state]);
3754                         continue;
3755 
3756                 default:
3757 #ifndef NDEBUG
3758                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
3759                             __FILE__, __LINE__, up_svcs_p[i]->gv_state);
3760 #endif
3761                         abort();
3762                 }
3763         }
3764 
3765         /*
3766          * In the seed repository, console-login is unsatisfiable because
3767          * services are missing.  To behave correctly in that case we don't want
3768          * to return false until manifest-import is online.
3769          */
3770 
3771         if (manifest_import_p == NULL) {
3772                 manifest_import_p = vertex_get_by_name(manifest_import);
3773 
3774                 if (manifest_import_p == NULL)
3775                         return (B_FALSE);
3776         }
3777 
3778         switch (manifest_import_p->gv_state) {
3779         case RESTARTER_STATE_ONLINE:
3780         case RESTARTER_STATE_DEGRADED:
3781         case RESTARTER_STATE_DISABLED:
3782         case RESTARTER_STATE_MAINT:
3783                 break;
3784 
3785         case RESTARTER_STATE_OFFLINE:
3786                 if (instance_satisfied(manifest_import_p, B_TRUE) == -1)
3787                         break;
3788                 /* FALLTHROUGH */
3789 
3790         case RESTARTER_STATE_UNINIT:
3791                 return (B_TRUE);
3792         }
3793 
3794         return (B_FALSE);
3795 }
3796 
3797 /*
3798  * Runs sulogin.  Returns
3799  *   0 - success
3800  *   EALREADY - sulogin is already running
3801  *   EBUSY - console-login is running
3802  */
3803 static int
3804 run_sulogin(const char *msg)
3805 {
3806         graph_vertex_t *v;
3807 
3808         assert(MUTEX_HELD(&dgraph_lock));
3809 
3810         if (sulogin_running)
3811                 return (EALREADY);
3812 
3813         v = vertex_get_by_name(console_login_fmri);
3814         if (v != NULL && inst_running(v))
3815                 return (EBUSY);
3816 
3817         sulogin_running = B_TRUE;
3818 
3819         MUTEX_UNLOCK(&dgraph_lock);
3820 
3821         fork_sulogin(B_FALSE, msg);
3822 
3823         MUTEX_LOCK(&dgraph_lock);
3824 
3825         sulogin_running = B_FALSE;
3826 
3827         if (console_login_ready) {
3828                 v = vertex_get_by_name(console_login_fmri);
3829 
3830                 if (v != NULL && v->gv_state == RESTARTER_STATE_OFFLINE) {
3831                         if (v->gv_start_f == NULL)
3832                                 vertex_send_event(v,
3833                                     RESTARTER_EVENT_TYPE_START);
3834                         else
3835                                 v->gv_start_f(v);
3836                 }
3837 
3838                 console_login_ready = B_FALSE;
3839         }
3840 
3841         return (0);
3842 }
3843 
3844 /*
3845  * The sulogin thread runs sulogin while can_come_up() is false.  run_sulogin()
3846  * keeps sulogin from stepping on console-login's toes.
3847  */
3848 /* ARGSUSED */
3849 static void *
3850 sulogin_thread(void *unused)
3851 {
3852         MUTEX_LOCK(&dgraph_lock);
3853 
3854         assert(sulogin_thread_running);
3855 
3856         do {
3857                 (void) run_sulogin("Console login service(s) cannot run\n");
3858         } while (!can_come_up());
3859 
3860         sulogin_thread_running = B_FALSE;
3861         MUTEX_UNLOCK(&dgraph_lock);
3862 
3863         return (NULL);
3864 }
3865 
3866 /* ARGSUSED */
3867 void *
3868 single_user_thread(void *unused)
3869 {
3870         uint_t left;
3871         scf_handle_t *h;
3872         scf_instance_t *inst;
3873         scf_property_t *prop;
3874         scf_value_t *val;
3875         const char *msg;
3876         char *buf;
3877         int r;
3878 
3879         MUTEX_LOCK(&single_user_thread_lock);
3880         single_user_thread_count++;
3881 
3882         if (!booting_to_single_user)
3883                 kill_user_procs();
3884 
3885         if (go_single_user_mode || booting_to_single_user) {
3886                 msg = "SINGLE USER MODE\n";
3887         } else {
3888                 assert(go_to_level1);
3889 
3890                 fork_rc_script('1', "start", B_TRUE);
3891 
3892                 uu_warn("The system is ready for administration.\n");
3893 
3894                 msg = "";
3895         }
3896 
3897         MUTEX_UNLOCK(&single_user_thread_lock);
3898 
3899         for (;;) {
3900                 MUTEX_LOCK(&dgraph_lock);
3901                 r = run_sulogin(msg);
3902                 MUTEX_UNLOCK(&dgraph_lock);
3903                 if (r == 0)
3904                         break;
3905 
3906                 assert(r == EALREADY || r == EBUSY);
3907 
3908                 left = 3;
3909                 while (left > 0)
3910                         left = sleep(left);
3911         }
3912 
3913         MUTEX_LOCK(&single_user_thread_lock);
3914 
3915         /*
3916          * If another single user thread has started, let it finish changing
3917          * the run level.
3918          */
3919         if (single_user_thread_count > 1) {
3920                 single_user_thread_count--;
3921                 MUTEX_UNLOCK(&single_user_thread_lock);
3922                 return (NULL);
3923         }
3924 
3925         h = libscf_handle_create_bound_loop();
3926         inst = scf_instance_create(h);
3927         prop = safe_scf_property_create(h);
3928         val = safe_scf_value_create(h);
3929         buf = startd_alloc(max_scf_fmri_size);
3930 
3931 lookup:
3932         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
3933             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
3934                 switch (scf_error()) {
3935                 case SCF_ERROR_NOT_FOUND:
3936                         r = libscf_create_self(h);
3937                         if (r == 0)
3938                                 goto lookup;
3939                         assert(r == ECONNABORTED);
3940                         /* FALLTHROUGH */
3941 
3942                 case SCF_ERROR_CONNECTION_BROKEN:
3943                         libscf_handle_rebind(h);
3944                         goto lookup;
3945 
3946                 case SCF_ERROR_INVALID_ARGUMENT:
3947                 case SCF_ERROR_CONSTRAINT_VIOLATED:
3948                 case SCF_ERROR_NOT_BOUND:
3949                 case SCF_ERROR_HANDLE_MISMATCH:
3950                 default:
3951                         bad_error("scf_handle_decode_fmri", scf_error());
3952                 }
3953         }
3954 
3955         MUTEX_LOCK(&dgraph_lock);
3956 
3957         r = scf_instance_delete_prop(inst, SCF_PG_OPTIONS_OVR,
3958             SCF_PROPERTY_MILESTONE);
3959         switch (r) {
3960         case 0:
3961         case ECANCELED:
3962                 break;
3963 
3964         case ECONNABORTED:
3965                 MUTEX_UNLOCK(&dgraph_lock);
3966                 libscf_handle_rebind(h);
3967                 goto lookup;
3968 
3969         case EPERM:
3970         case EACCES:
3971         case EROFS:
3972                 log_error(LOG_WARNING, "Could not clear temporary milestone: "
3973                     "%s.\n", strerror(r));
3974                 break;
3975 
3976         default:
3977                 bad_error("scf_instance_delete_prop", r);
3978         }
3979 
3980         MUTEX_UNLOCK(&dgraph_lock);
3981 
3982         r = libscf_get_milestone(inst, prop, val, buf, max_scf_fmri_size);
3983         switch (r) {
3984         case ECANCELED:
3985         case ENOENT:
3986         case EINVAL:
3987                 (void) strcpy(buf, "all");
3988                 /* FALLTHROUGH */
3989 
3990         case 0:
3991                 uu_warn("Returning to milestone %s.\n", buf);
3992                 break;
3993 
3994         case ECONNABORTED:
3995                 libscf_handle_rebind(h);
3996                 goto lookup;
3997 
3998         default:
3999                 bad_error("libscf_get_milestone", r);
4000         }
4001 
4002         r = dgraph_set_milestone(buf, h, B_FALSE);
4003         switch (r) {
4004         case 0:
4005         case ECONNRESET:
4006         case EALREADY:
4007         case EINVAL:
4008         case ENOENT:
4009                 break;
4010 
4011         default:
4012                 bad_error("dgraph_set_milestone", r);
4013         }
4014 
4015         /*
4016          * See graph_runlevel_changed().
4017          */
4018         MUTEX_LOCK(&dgraph_lock);
4019         utmpx_set_runlevel(target_milestone_as_runlevel(), 'S', B_TRUE);
4020         MUTEX_UNLOCK(&dgraph_lock);
4021 
4022         startd_free(buf, max_scf_fmri_size);
4023         scf_value_destroy(val);
4024         scf_property_destroy(prop);
4025         scf_instance_destroy(inst);
4026         scf_handle_destroy(h);
4027 
4028         /*
4029          * We'll give ourselves 3 seconds to respond to all of the enablings
4030          * that setting the milestone should have created before checking
4031          * whether to run sulogin.
4032          */
4033         left = 3;
4034         while (left > 0)
4035                 left = sleep(left);
4036 
4037         MUTEX_LOCK(&dgraph_lock);
4038         /*
4039          * Clearing these variables will allow the sulogin thread to run.  We
4040          * check here in case there aren't any more state updates anytime soon.
4041          */
4042         go_to_level1 = go_single_user_mode = booting_to_single_user = B_FALSE;
4043         if (!sulogin_thread_running && !can_come_up()) {
4044                 (void) startd_thread_create(sulogin_thread, NULL);
4045                 sulogin_thread_running = B_TRUE;
4046         }
4047         MUTEX_UNLOCK(&dgraph_lock);
4048         single_user_thread_count--;
4049         MUTEX_UNLOCK(&single_user_thread_lock);
4050         return (NULL);
4051 }
4052 
4053 
4054 /*
4055  * Dependency graph operations API.  These are handle-independent thread-safe
4056  * graph manipulation functions which are the entry points for the event
4057  * threads below.
4058  */
4059 
4060 /*
4061  * If a configured vertex exists for inst_fmri, return EEXIST.  If no vertex
4062  * exists for inst_fmri, add one.  Then fetch the restarter from inst, make
4063  * this vertex dependent on it, and send _ADD_INSTANCE to the restarter.
4064  * Fetch whether the instance should be enabled from inst and send _ENABLE or
4065  * _DISABLE as appropriate.  Finally rummage through inst's dependency
4066  * property groups and add vertices and edges as appropriate.  If anything
4067  * goes wrong after sending _ADD_INSTANCE, send _ADMIN_MAINT_ON to put the
4068  * instance in maintenance.  Don't send _START or _STOP until we get a state
4069  * update in case we're being restarted and the service is already running.
4070  *
4071  * To support booting to a milestone, we must also make sure all dependencies
4072  * encountered are configured, if they exist in the repository.
4073  *
4074  * Returns 0 on success, ECONNABORTED on repository disconnection, EINVAL if
4075  * inst_fmri is an invalid (or not canonical) FMRI, ECANCELED if inst is
4076  * deleted, or EEXIST if a configured vertex for inst_fmri already exists.
4077  */
4078 int
4079 dgraph_add_instance(const char *inst_fmri, scf_instance_t *inst,
4080     boolean_t lock_graph)
4081 {
4082         graph_vertex_t *v;
4083         int err;
4084 
4085         if (strcmp(inst_fmri, SCF_SERVICE_STARTD) == 0)
4086                 return (0);
4087 
4088         /* Check for a vertex for inst_fmri. */
4089         if (lock_graph) {
4090                 MUTEX_LOCK(&dgraph_lock);
4091         } else {
4092                 assert(MUTEX_HELD(&dgraph_lock));
4093         }
4094 
4095         v = vertex_get_by_name(inst_fmri);
4096 
4097         if (v != NULL) {
4098                 assert(v->gv_type == GVT_INST);
4099 
4100                 if (v->gv_flags & GV_CONFIGURED) {
4101                         if (lock_graph)
4102                                 MUTEX_UNLOCK(&dgraph_lock);
4103                         return (EEXIST);
4104                 }
4105         } else {
4106                 /* Add the vertex. */
4107                 err = graph_insert_vertex_unconfigured(inst_fmri, GVT_INST, 0,
4108                     RERR_NONE, &v);
4109                 if (err != 0) {
4110                         assert(err == EINVAL);
4111                         if (lock_graph)
4112                                 MUTEX_UNLOCK(&dgraph_lock);
4113                         return (EINVAL);
4114                 }
4115         }
4116 
4117         err = configure_vertex(v, inst);
4118 
4119         if (lock_graph)
4120                 MUTEX_UNLOCK(&dgraph_lock);
4121 
4122         return (err);
4123 }
4124 
4125 /*
4126  * Locate the vertex for this property group's instance.  If it doesn't exist
4127  * or is unconfigured, call dgraph_add_instance() & return.  Otherwise fetch
4128  * the restarter for the instance, and if it has changed, send
4129  * _REMOVE_INSTANCE to the old restarter, remove the dependency, make sure the
4130  * new restarter has a vertex, add a new dependency, and send _ADD_INSTANCE to
4131  * the new restarter.  Then fetch whether the instance should be enabled, and
4132  * if it is different from what we had, or if we changed the restarter, send
4133  * the appropriate _ENABLE or _DISABLE command.
4134  *
4135  * Returns 0 on success, ENOTSUP if the pg's parent is not an instance,
4136  * ECONNABORTED on repository disconnection, ECANCELED if the instance is
4137  * deleted, or -1 if the instance's general property group is deleted or if
4138  * its enabled property is misconfigured.
4139  */
4140 static int
4141 dgraph_update_general(scf_propertygroup_t *pg)
4142 {
4143         scf_handle_t *h;
4144         scf_instance_t *inst;
4145         char *fmri;
4146         char *restarter_fmri;
4147         graph_vertex_t *v;
4148         int err;
4149         int enabled, enabled_ovr;
4150         int oldflags;
4151 
4152         /* Find the vertex for this service */
4153         h = scf_pg_handle(pg);
4154 
4155         inst = safe_scf_instance_create(h);
4156 
4157         if (scf_pg_get_parent_instance(pg, inst) != 0) {
4158                 switch (scf_error()) {
4159                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4160                         return (ENOTSUP);
4161 
4162                 case SCF_ERROR_CONNECTION_BROKEN:
4163                 default:
4164                         return (ECONNABORTED);
4165 
4166                 case SCF_ERROR_DELETED:
4167                         return (0);
4168 
4169                 case SCF_ERROR_NOT_SET:
4170                         bad_error("scf_pg_get_parent_instance", scf_error());
4171                 }
4172         }
4173 
4174         err = libscf_instance_get_fmri(inst, &fmri);
4175         switch (err) {
4176         case 0:
4177                 break;
4178 
4179         case ECONNABORTED:
4180                 scf_instance_destroy(inst);
4181                 return (ECONNABORTED);
4182 
4183         case ECANCELED:
4184                 scf_instance_destroy(inst);
4185                 return (0);
4186 
4187         default:
4188                 bad_error("libscf_instance_get_fmri", err);
4189         }
4190 
4191         log_framework(LOG_DEBUG,
4192             "Graph engine: Reloading general properties for %s.\n", fmri);
4193 
4194         MUTEX_LOCK(&dgraph_lock);
4195 
4196         v = vertex_get_by_name(fmri);
4197         if (v == NULL || !(v->gv_flags & GV_CONFIGURED)) {
4198                 /* Will get the up-to-date properties. */
4199                 MUTEX_UNLOCK(&dgraph_lock);
4200                 err = dgraph_add_instance(fmri, inst, B_TRUE);
4201                 startd_free(fmri, max_scf_fmri_size);
4202                 scf_instance_destroy(inst);
4203                 return (err == ECANCELED ? 0 : err);
4204         }
4205 
4206         /* Read enabled & restarter from repository. */
4207         restarter_fmri = startd_alloc(max_scf_value_size);
4208         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
4209             &enabled_ovr, &restarter_fmri);
4210         if (err != 0 || enabled == -1) {
4211                 MUTEX_UNLOCK(&dgraph_lock);
4212                 scf_instance_destroy(inst);
4213                 startd_free(fmri, max_scf_fmri_size);
4214 
4215                 switch (err) {
4216                 case ENOENT:
4217                 case 0:
4218                         startd_free(restarter_fmri, max_scf_value_size);
4219                         return (-1);
4220 
4221                 case ECONNABORTED:
4222                 case ECANCELED:
4223                         startd_free(restarter_fmri, max_scf_value_size);
4224                         return (err);
4225 
4226                 default:
4227                         bad_error("libscf_get_basic_instance_data", err);
4228                 }
4229         }
4230 
4231         oldflags = v->gv_flags;
4232         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
4233             (enabled ? GV_ENBLD_NOOVR : 0);
4234 
4235         if (enabled_ovr != -1)
4236                 enabled = enabled_ovr;
4237 
4238         /*
4239          * If GV_ENBLD_NOOVR has changed, then we need to re-evaluate the
4240          * subgraph.
4241          */
4242         if (milestone > MILESTONE_NONE && v->gv_flags != oldflags)
4243                 (void) eval_subgraph(v, h);
4244 
4245         scf_instance_destroy(inst);
4246 
4247         /* Ignore restarter change for now. */
4248 
4249         startd_free(restarter_fmri, max_scf_value_size);
4250         startd_free(fmri, max_scf_fmri_size);
4251 
4252         /*
4253          * Always send _ENABLE or _DISABLE.  We could avoid this if the
4254          * restarter didn't change and the enabled value didn't change, but
4255          * that's not easy to check and improbable anyway, so we'll just do
4256          * this.
4257          */
4258         graph_enable_by_vertex(v, enabled, 1);
4259 
4260         MUTEX_UNLOCK(&dgraph_lock);
4261 
4262         return (0);
4263 }
4264 
4265 /*
4266  * Delete all of the property group dependencies of v, update inst's running
4267  * snapshot, and add the dependencies in the new snapshot.  If any of the new
4268  * dependencies would create a cycle, send _ADMIN_MAINT_ON.  Otherwise
4269  * reevaluate v's dependencies, send _START or _STOP as appropriate, and do
4270  * the same for v's dependents.
4271  *
4272  * Returns
4273  *   0 - success
4274  *   ECONNABORTED - repository connection broken
4275  *   ECANCELED - inst was deleted
4276  *   EINVAL - inst is invalid (e.g., missing general/enabled)
4277  *   -1 - libscf_snapshots_refresh() failed
4278  */
4279 static int
4280 dgraph_refresh_instance(graph_vertex_t *v, scf_instance_t *inst)
4281 {
4282         int r;
4283         int enabled;
4284         int32_t tset;
4285 
4286         assert(MUTEX_HELD(&dgraph_lock));
4287         assert(v->gv_type == GVT_INST);
4288 
4289         /* Only refresh services with valid general/enabled properties. */
4290         r = libscf_get_basic_instance_data(scf_instance_handle(inst), inst,
4291             v->gv_name, &enabled, NULL, NULL);
4292         switch (r) {
4293         case 0:
4294                 break;
4295 
4296         case ECONNABORTED:
4297         case ECANCELED:
4298                 return (r);
4299 
4300         case ENOENT:
4301                 log_framework(LOG_DEBUG,
4302                     "Ignoring %s because it has no general property group.\n",
4303                     v->gv_name);
4304                 return (EINVAL);
4305 
4306         default:
4307                 bad_error("libscf_get_basic_instance_data", r);
4308         }
4309 
4310         if ((tset = libscf_get_stn_tset(inst)) == -1) {
4311                 log_framework(LOG_WARNING,
4312                     "Failed to get notification parameters for %s: %s\n",
4313                     v->gv_name, scf_strerror(scf_error()));
4314                 tset = 0;
4315         }
4316         v->gv_stn_tset = tset;
4317         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
4318                 stn_global = tset;
4319 
4320         if (enabled == -1)
4321                 return (EINVAL);
4322 
4323         r = libscf_snapshots_refresh(inst, v->gv_name);
4324         if (r != 0) {
4325                 if (r != -1)
4326                         bad_error("libscf_snapshots_refresh", r);
4327 
4328                 /* error logged */
4329                 return (r);
4330         }
4331 
4332         r = refresh_vertex(v, inst);
4333         if (r != 0 && r != ECONNABORTED)
4334                 bad_error("refresh_vertex", r);
4335         return (r);
4336 }
4337 
4338 /*
4339  * Returns true only if none of this service's dependents are 'up' -- online
4340  * or degraded (offline is considered down in this situation). This function
4341  * is somehow similar to is_nonsubgraph_leaf() but works on subtrees.
4342  */
4343 static boolean_t
4344 insubtree_dependents_down(graph_vertex_t *v)
4345 {
4346         graph_vertex_t *vv;
4347         graph_edge_t *e;
4348 
4349         assert(MUTEX_HELD(&dgraph_lock));
4350 
4351         for (e = uu_list_first(v->gv_dependents); e != NULL;
4352             e = uu_list_next(v->gv_dependents, e)) {
4353                 vv = e->ge_vertex;
4354                 if (vv->gv_type == GVT_INST) {
4355                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4356                                 continue;
4357 
4358                         if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4359                                 continue;
4360 
4361                         if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4362                             (vv->gv_state == RESTARTER_STATE_DEGRADED))
4363                                 return (B_FALSE);
4364                 } else {
4365                         /*
4366                          * Skip all excluded dependents and decide whether
4367                          * to offline the service based on the restart_on
4368                          * on attribute.
4369                          */
4370                         if (is_depgrp_bypassed(vv))
4371                                 continue;
4372 
4373                         /*
4374                          * For dependency groups or service vertices, keep
4375                          * traversing to see if instances are running.
4376                          */
4377                         if (insubtree_dependents_down(vv) == B_FALSE)
4378                                 return (B_FALSE);
4379                 }
4380         }
4381 
4382         return (B_TRUE);
4383 }
4384 
4385 /*
4386  * Returns true only if none of this service's dependents are 'up' -- online,
4387  * degraded, or offline.
4388  */
4389 static int
4390 is_nonsubgraph_leaf(graph_vertex_t *v)
4391 {
4392         graph_vertex_t *vv;
4393         graph_edge_t *e;
4394 
4395         assert(MUTEX_HELD(&dgraph_lock));
4396 
4397         for (e = uu_list_first(v->gv_dependents);
4398             e != NULL;
4399             e = uu_list_next(v->gv_dependents, e)) {
4400 
4401                 vv = e->ge_vertex;
4402                 if (vv->gv_type == GVT_INST) {
4403                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4404                                 continue;
4405 
4406                         if (vv->gv_flags & GV_INSUBGRAPH)
4407                                 continue;
4408 
4409                         if (up_state(vv->gv_state))
4410                                 return (0);
4411                 } else {
4412                         /*
4413                          * For dependency group or service vertices, keep
4414                          * traversing to see if instances are running.
4415                          *
4416                          * We should skip exclude_all dependencies otherwise
4417                          * the vertex will never be considered as a leaf
4418                          * if the dependent is offline. The main reason for
4419                          * this is that disable_nonsubgraph_leaves() skips
4420                          * exclusion dependencies.
4421                          */
4422                         if (vv->gv_type == GVT_GROUP &&
4423                             vv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4424                                 continue;
4425 
4426                         if (!is_nonsubgraph_leaf(vv))
4427                                 return (0);
4428                 }
4429         }
4430 
4431         return (1);
4432 }
4433 
4434 /*
4435  * Disable v temporarily.  Attempt to do this by setting its enabled override
4436  * property in the repository.  If that fails, send a _DISABLE command.
4437  * Returns 0 on success and ECONNABORTED if the repository connection is
4438  * broken.
4439  */
4440 static int
4441 disable_service_temporarily(graph_vertex_t *v, scf_handle_t *h)
4442 {
4443         const char * const emsg = "Could not temporarily disable %s because "
4444             "%s.  Will stop service anyways.  Repository status for the "
4445             "service may be inaccurate.\n";
4446         const char * const emsg_cbroken =
4447             "the repository connection was broken";
4448 
4449         scf_instance_t *inst;
4450         int r;
4451 
4452         inst = scf_instance_create(h);
4453         if (inst == NULL) {
4454                 char buf[100];
4455 
4456                 (void) snprintf(buf, sizeof (buf),
4457                     "scf_instance_create() failed (%s)",
4458                     scf_strerror(scf_error()));
4459                 log_error(LOG_WARNING, emsg, v->gv_name, buf);
4460 
4461                 graph_enable_by_vertex(v, 0, 0);
4462                 return (0);
4463         }
4464 
4465         r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
4466             NULL, NULL, SCF_DECODE_FMRI_EXACT);
4467         if (r != 0) {
4468                 switch (scf_error()) {
4469                 case SCF_ERROR_CONNECTION_BROKEN:
4470                         log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4471                         graph_enable_by_vertex(v, 0, 0);
4472                         return (ECONNABORTED);
4473 
4474                 case SCF_ERROR_NOT_FOUND:
4475                         return (0);
4476 
4477                 case SCF_ERROR_HANDLE_MISMATCH:
4478                 case SCF_ERROR_INVALID_ARGUMENT:
4479                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4480                 case SCF_ERROR_NOT_BOUND:
4481                 default:
4482                         bad_error("scf_handle_decode_fmri",
4483                             scf_error());
4484                 }
4485         }
4486 
4487         r = libscf_set_enable_ovr(inst, 0);
4488         switch (r) {
4489         case 0:
4490                 scf_instance_destroy(inst);
4491                 return (0);
4492 
4493         case ECANCELED:
4494                 scf_instance_destroy(inst);
4495                 return (0);
4496 
4497         case ECONNABORTED:
4498                 log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4499                 graph_enable_by_vertex(v, 0, 0);
4500                 return (ECONNABORTED);
4501 
4502         case EPERM:
4503                 log_error(LOG_WARNING, emsg, v->gv_name,
4504                     "the repository denied permission");
4505                 graph_enable_by_vertex(v, 0, 0);
4506                 return (0);
4507 
4508         case EROFS:
4509                 log_error(LOG_WARNING, emsg, v->gv_name,
4510                     "the repository is read-only");
4511                 graph_enable_by_vertex(v, 0, 0);
4512                 return (0);
4513 
4514         default:
4515                 bad_error("libscf_set_enable_ovr", r);
4516                 /* NOTREACHED */
4517         }
4518 }
4519 
4520 /*
4521  * Of the transitive instance dependencies of v, offline those which are
4522  * in the subtree and which are leaves (i.e., have no dependents which are
4523  * "up").
4524  */
4525 void
4526 offline_subtree_leaves(graph_vertex_t *v, void *arg)
4527 {
4528         assert(MUTEX_HELD(&dgraph_lock));
4529 
4530         /* If v isn't an instance, recurse on its dependencies. */
4531         if (v->gv_type != GVT_INST) {
4532                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4533                 return;
4534         }
4535 
4536         /*
4537          * If v is not in the subtree, so should all of its dependencies,
4538          * so do nothing.
4539          */
4540         if ((v->gv_flags & GV_TOOFFLINE) == 0)
4541                 return;
4542 
4543         /* If v isn't a leaf because it's already down, recurse. */
4544         if (!inst_running(v) && (v->gv_flags & GV_TODISABLE) == 0) {
4545                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4546                 return;
4547         }
4548 
4549         /* if v is a leaf, offline it or disable it if it's the last one */
4550         if (insubtree_dependents_down(v) == B_TRUE) {
4551                 if (v->gv_flags & GV_TODISABLE)
4552                         vertex_send_event(v,
4553                             RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
4554                 else
4555                         offline_vertex(v);
4556         }
4557 }
4558 
4559 void
4560 graph_offline_subtree_leaves(graph_vertex_t *v, void *h)
4561 {
4562         graph_walk_dependencies(v, offline_subtree_leaves, (void *)h);
4563 }
4564 
4565 
4566 /*
4567  * Of the transitive instance dependencies of v, disable those which are not
4568  * in the subgraph and which are leaves (i.e., have no dependents which are
4569  * "up").
4570  */
4571 static void
4572 disable_nonsubgraph_leaves(graph_vertex_t *v, void *arg)
4573 {
4574         assert(MUTEX_HELD(&dgraph_lock));
4575 
4576         /*
4577          * We must skip exclusion dependencies because they are allowed to
4578          * complete dependency cycles.  This is correct because A's exclusion
4579          * dependency on B doesn't bear on the order in which they should be
4580          * stopped.  Indeed, the exclusion dependency should guarantee that
4581          * they are never online at the same time.
4582          */
4583         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4584                 return;
4585 
4586         /* If v isn't an instance, recurse on its dependencies. */
4587         if (v->gv_type != GVT_INST)
4588                 goto recurse;
4589 
4590         if ((v->gv_flags & GV_CONFIGURED) == 0)
4591                 /*
4592                  * Unconfigured instances should have no dependencies, but in
4593                  * case they ever get them,
4594                  */
4595                 goto recurse;
4596 
4597         /*
4598          * If v is in the subgraph, so should all of its dependencies, so do
4599          * nothing.
4600          */
4601         if (v->gv_flags & GV_INSUBGRAPH)
4602                 return;
4603 
4604         /* If v isn't a leaf because it's already down, recurse. */
4605         if (!up_state(v->gv_state))
4606                 goto recurse;
4607 
4608         /* If v is disabled but not down yet, be patient. */
4609         if ((v->gv_flags & GV_ENABLED) == 0)
4610                 return;
4611 
4612         /* If v is a leaf, disable it. */
4613         if (is_nonsubgraph_leaf(v))
4614                 (void) disable_service_temporarily(v, (scf_handle_t *)arg);
4615 
4616         return;
4617 
4618 recurse:
4619         graph_walk_dependencies(v, disable_nonsubgraph_leaves, arg);
4620 }
4621 
4622 static int
4623 stn_restarter_state(restarter_instance_state_t rstate)
4624 {
4625         static const struct statemap {
4626                 restarter_instance_state_t restarter_state;
4627                 int scf_state;
4628         } map[] = {
4629                 { RESTARTER_STATE_UNINIT, SCF_STATE_UNINIT },
4630                 { RESTARTER_STATE_MAINT, SCF_STATE_MAINT },
4631                 { RESTARTER_STATE_OFFLINE, SCF_STATE_OFFLINE },
4632                 { RESTARTER_STATE_DISABLED, SCF_STATE_DISABLED },
4633                 { RESTARTER_STATE_ONLINE, SCF_STATE_ONLINE },
4634                 { RESTARTER_STATE_DEGRADED, SCF_STATE_DEGRADED }
4635         };
4636 
4637         int i;
4638 
4639         for (i = 0; i < sizeof (map) / sizeof (map[0]); i++) {
4640                 if (rstate == map[i].restarter_state)
4641                         return (map[i].scf_state);
4642         }
4643 
4644         return (-1);
4645 }
4646 
4647 /*
4648  * State transition counters
4649  * Not incremented atomically - indicative only
4650  */
4651 static uint64_t stev_ct_maint;
4652 static uint64_t stev_ct_hwerr;
4653 static uint64_t stev_ct_service;
4654 static uint64_t stev_ct_global;
4655 static uint64_t stev_ct_noprefs;
4656 static uint64_t stev_ct_from_uninit;
4657 static uint64_t stev_ct_bad_state;
4658 static uint64_t stev_ct_ovr_prefs;
4659 
4660 static void
4661 dgraph_state_transition_notify(graph_vertex_t *v,
4662     restarter_instance_state_t old_state, restarter_str_t reason)
4663 {
4664         restarter_instance_state_t new_state = v->gv_state;
4665         int stn_transition, maint;
4666         int from, to;
4667         nvlist_t *attr;
4668         fmev_pri_t pri = FMEV_LOPRI;
4669         int raise = 0;
4670 
4671         if ((from = stn_restarter_state(old_state)) == -1 ||
4672             (to = stn_restarter_state(new_state)) == -1) {
4673                 stev_ct_bad_state++;
4674                 return;
4675         }
4676 
4677         stn_transition = from << 16 | to;
4678 
4679         maint = (to == SCF_STATE_MAINT || from == SCF_STATE_MAINT);
4680 
4681         if (maint) {
4682                 /*
4683                  * All transitions to/from maintenance state must raise
4684                  * an event.
4685                  */
4686                 raise++;
4687                 pri = FMEV_HIPRI;
4688                 stev_ct_maint++;
4689         } else if (reason == restarter_str_ct_ev_hwerr) {
4690                 /*
4691                  * All transitions caused by hardware fault must raise
4692                  * an event
4693                  */
4694                 raise++;
4695                 pri = FMEV_HIPRI;
4696                 stev_ct_hwerr++;
4697         } else if (stn_transition & v->gv_stn_tset) {
4698                 /*
4699                  * Specifically enabled event.
4700                  */
4701                 raise++;
4702                 stev_ct_service++;
4703         } else if (from == SCF_STATE_UNINIT) {
4704                 /*
4705                  * Only raise these if specifically selected above.
4706                  */
4707                 stev_ct_from_uninit++;
4708         } else if (stn_transition & stn_global &&
4709             (IS_ENABLED(v) == 1 || to == SCF_STATE_DISABLED)) {
4710                 raise++;
4711                 stev_ct_global++;
4712         } else {
4713                 stev_ct_noprefs++;
4714         }
4715 
4716         if (info_events_all) {
4717                 stev_ct_ovr_prefs++;
4718                 raise++;
4719         }
4720         if (!raise)
4721                 return;
4722 
4723         if (nvlist_alloc(&attr, NV_UNIQUE_NAME, 0) != 0 ||
4724             nvlist_add_string(attr, "fmri", v->gv_name) != 0 ||
4725             nvlist_add_uint32(attr, "reason-version",
4726             restarter_str_version()) || nvlist_add_string(attr, "reason-short",
4727             restarter_get_str_short(reason)) != 0 ||
4728             nvlist_add_string(attr, "reason-long",
4729             restarter_get_str_long(reason)) != 0 ||
4730             nvlist_add_int32(attr, "transition", stn_transition) != 0) {
4731                 log_framework(LOG_WARNING,
4732                     "FMEV: %s could not create nvlist for transition "
4733                     "event: %s\n", v->gv_name, strerror(errno));
4734                 nvlist_free(attr);
4735                 return;
4736         }
4737 
4738         if (fmev_rspublish_nvl(FMEV_RULESET_SMF, "state-transition",
4739             instance_state_str[new_state], pri, attr) != FMEV_SUCCESS) {
4740                 log_framework(LOG_DEBUG,
4741                     "FMEV: %s failed to publish transition event: %s\n",
4742                     v->gv_name, fmev_strerror(fmev_errno));
4743                 nvlist_free(attr);
4744         }
4745 }
4746 
4747 /*
4748  * Find the vertex for inst_name.  If it doesn't exist, return ENOENT.
4749  * Otherwise set its state to state.  If the instance has entered a state
4750  * which requires automatic action, take it (Uninitialized: do
4751  * dgraph_refresh_instance() without the snapshot update.  Disabled: if the
4752  * instance should be enabled, send _ENABLE.  Offline: if the instance should
4753  * be disabled, send _DISABLE, and if its dependencies are satisfied, send
4754  * _START.  Online, Degraded: if the instance wasn't running, update its start
4755  * snapshot.  Maintenance: no action.)
4756  *
4757  * Also fails with ECONNABORTED, or EINVAL if state is invalid.
4758  */
4759 static int
4760 dgraph_set_instance_state(scf_handle_t *h, const char *inst_name,
4761     protocol_states_t *states)
4762 {
4763         graph_vertex_t *v;
4764         int err = 0;
4765         restarter_instance_state_t old_state;
4766         restarter_instance_state_t state = states->ps_state;
4767         restarter_error_t serr = states->ps_err;
4768 
4769         MUTEX_LOCK(&dgraph_lock);
4770 
4771         v = vertex_get_by_name(inst_name);
4772         if (v == NULL) {
4773                 MUTEX_UNLOCK(&dgraph_lock);
4774                 return (ENOENT);
4775         }
4776 
4777         assert(v->gv_type == GVT_INST);
4778 
4779         switch (state) {
4780         case RESTARTER_STATE_UNINIT:
4781         case RESTARTER_STATE_DISABLED:
4782         case RESTARTER_STATE_OFFLINE:
4783         case RESTARTER_STATE_ONLINE:
4784         case RESTARTER_STATE_DEGRADED:
4785         case RESTARTER_STATE_MAINT:
4786                 break;
4787 
4788         default:
4789                 MUTEX_UNLOCK(&dgraph_lock);
4790                 return (EINVAL);
4791         }
4792 
4793         log_framework(LOG_DEBUG, "Graph noting %s %s -> %s.\n", v->gv_name,
4794             instance_state_str[v->gv_state], instance_state_str[state]);
4795 
4796         old_state = v->gv_state;
4797         v->gv_state = state;
4798 
4799         v->gv_reason = states->ps_reason;
4800         err = gt_transition(h, v, serr, old_state);
4801         if (err == 0 && v->gv_state != old_state) {
4802                 dgraph_state_transition_notify(v, old_state, states->ps_reason);
4803         }
4804 
4805         MUTEX_UNLOCK(&dgraph_lock);
4806         return (err);
4807 }
4808 
4809 /*
4810  * Handle state changes during milestone shutdown.  See
4811  * dgraph_set_milestone().  If the repository connection is broken,
4812  * ECONNABORTED will be returned, though a _DISABLE command will be sent for
4813  * the vertex anyway.
4814  */
4815 int
4816 vertex_subgraph_dependencies_shutdown(scf_handle_t *h, graph_vertex_t *v,
4817     restarter_instance_state_t old_state)
4818 {
4819         int was_up, now_up;
4820         int ret = 0;
4821 
4822         assert(v->gv_type == GVT_INST);
4823 
4824         /* Don't care if we're not going to a milestone. */
4825         if (milestone == NULL)
4826                 return (0);
4827 
4828         /* Don't care if we already finished coming down. */
4829         if (non_subgraph_svcs == 0)
4830                 return (0);
4831 
4832         /* Don't care if the service is in the subgraph. */
4833         if (v->gv_flags & GV_INSUBGRAPH)
4834                 return (0);
4835 
4836         /*
4837          * Update non_subgraph_svcs.  It is the number of non-subgraph
4838          * services which are in online, degraded, or offline.
4839          */
4840 
4841         was_up = up_state(old_state);
4842         now_up = up_state(v->gv_state);
4843 
4844         if (!was_up && now_up) {
4845                 ++non_subgraph_svcs;
4846         } else if (was_up && !now_up) {
4847                 --non_subgraph_svcs;
4848 
4849                 if (non_subgraph_svcs == 0) {
4850                         if (halting != -1) {
4851                                 do_uadmin();
4852                         } else if (go_single_user_mode || go_to_level1) {
4853                                 (void) startd_thread_create(single_user_thread,
4854                                     NULL);
4855                         }
4856                         return (0);
4857                 }
4858         }
4859 
4860         /* If this service is a leaf, it should be disabled. */
4861         if ((v->gv_flags & GV_ENABLED) && is_nonsubgraph_leaf(v)) {
4862                 int r;
4863 
4864                 r = disable_service_temporarily(v, h);
4865                 switch (r) {
4866                 case 0:
4867                         break;
4868 
4869                 case ECONNABORTED:
4870                         ret = ECONNABORTED;
4871                         break;
4872 
4873                 default:
4874                         bad_error("disable_service_temporarily", r);
4875                 }
4876         }
4877 
4878         /*
4879          * If the service just came down, propagate the disable to the newly
4880          * exposed leaves.
4881          */
4882         if (was_up && !now_up)
4883                 graph_walk_dependencies(v, disable_nonsubgraph_leaves,
4884                     (void *)h);
4885 
4886         return (ret);
4887 }
4888 
4889 /*
4890  * Decide whether to start up an sulogin thread after a service is
4891  * finished changing state.  Only need to do the full can_come_up()
4892  * evaluation if an instance is changing state, we're not halfway through
4893  * loading the thread, and we aren't shutting down or going to the single
4894  * user milestone.
4895  */
4896 void
4897 graph_transition_sulogin(restarter_instance_state_t state,
4898     restarter_instance_state_t old_state)
4899 {
4900         assert(MUTEX_HELD(&dgraph_lock));
4901 
4902         if (state != old_state && st->st_load_complete &&
4903             !go_single_user_mode && !go_to_level1 &&
4904             halting == -1) {
4905                 if (!sulogin_thread_running && !can_come_up()) {
4906                         (void) startd_thread_create(sulogin_thread, NULL);
4907                         sulogin_thread_running = B_TRUE;
4908                 }
4909         }
4910 }
4911 
4912 /*
4913  * Propagate a start, stop event, or a satisfiability event.
4914  *
4915  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4916  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4917  * full dependent graph to check for newly satisfied nodes.  This is
4918  * necessary for cases when non-direct dependents may be effected but direct
4919  * dependents may not (e.g. for optional_all evaluations, see the
4920  * propagate_satbility() comments).
4921  *
4922  * PROPAGATE_SAT should be used whenever a non-running service moves into
4923  * a state which can satisfy optional dependencies, like disabled or
4924  * maintenance.
4925  */
4926 void
4927 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4928     restarter_error_t rerr)
4929 {
4930         if (type == PROPAGATE_STOP) {
4931                 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4932         } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4933                 graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
4934 
4935                 if (type == PROPAGATE_SAT)
4936                         propagate_satbility(v);
4937         } else {
4938 #ifndef NDEBUG
4939                 uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4940                     __LINE__, type);
4941 #endif
4942                 abort();
4943         }
4944 }
4945 
4946 /*
4947  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4948  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4949  * all property group dependencies, and the dependency on the restarter,
4950  * disposing of vertices as appropriate.  If other vertices depend on this
4951  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4952  * returns 0.
4953  */
4954 static int
4955 dgraph_remove_instance(const char *fmri, scf_handle_t *h)
4956 {
4957         graph_vertex_t *v;
4958         graph_edge_t *e;
4959         uu_list_t *old_deps;
4960         int err;
4961 
4962         log_framework(LOG_DEBUG, "Graph engine: Removing %s.\n", fmri);
4963 
4964         MUTEX_LOCK(&dgraph_lock);
4965 
4966         v = vertex_get_by_name(fmri);
4967         if (v == NULL) {
4968                 MUTEX_UNLOCK(&dgraph_lock);
4969                 return (0);
4970         }
4971 
4972         /* Send restarter delete event. */
4973         if (v->gv_flags & GV_CONFIGURED)
4974                 graph_unset_restarter(v);
4975 
4976         if (milestone > MILESTONE_NONE) {
4977                 /*
4978                  * Make a list of v's current dependencies so we can
4979                  * reevaluate their GV_INSUBGRAPH flags after the dependencies
4980                  * are removed.
4981                  */
4982                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
4983 
4984                 err = uu_list_walk(v->gv_dependencies,
4985                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
4986                 assert(err == 0);
4987         }
4988 
4989         delete_instance_dependencies(v, B_TRUE);
4990 
4991         /*
4992          * Deleting an instance can both satisfy and unsatisfy dependencies,
4993          * depending on their type.  First propagate the stop as a RERR_RESTART
4994          * event -- deletion isn't a fault, just a normal stop.  This gives
4995          * dependent services the chance to do a clean shutdown.  Then, mark
4996          * the service as unconfigured and propagate the start event for the
4997          * optional_all dependencies that might have become satisfied.
4998          */
4999         graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
5000 
5001         v->gv_flags &= ~GV_CONFIGURED;
5002         v->gv_flags &= ~GV_DEATHROW;
5003 
5004         graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
5005         propagate_satbility(v);
5006 
5007         /*
5008          * If there are no (non-service) dependents, the vertex can be
5009          * completely removed.
5010          */
5011         if (v != milestone && v->gv_refs == 0 &&
5012             uu_list_numnodes(v->gv_dependents) == 1)
5013                 remove_inst_vertex(v);
5014 
5015         if (milestone > MILESTONE_NONE) {
5016                 void *cookie = NULL;
5017 
5018                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5019                         v = e->ge_vertex;
5020 
5021                         if (vertex_unref(v) == VERTEX_INUSE)
5022                                 while (eval_subgraph(v, h) == ECONNABORTED)
5023                                         libscf_handle_rebind(h);
5024 
5025                         startd_free(e, sizeof (*e));
5026                 }
5027 
5028                 uu_list_destroy(old_deps);
5029         }
5030 
5031         MUTEX_UNLOCK(&dgraph_lock);
5032 
5033         return (0);
5034 }
5035 
5036 /*
5037  * Return the eventual (maybe current) milestone in the form of a
5038  * legacy runlevel.
5039  */
5040 static char
5041 target_milestone_as_runlevel()
5042 {
5043         assert(MUTEX_HELD(&dgraph_lock));
5044 
5045         if (milestone == NULL)
5046                 return ('3');
5047         else if (milestone == MILESTONE_NONE)
5048                 return ('0');
5049 
5050         if (strcmp(milestone->gv_name, multi_user_fmri) == 0)
5051                 return ('2');
5052         else if (strcmp(milestone->gv_name, single_user_fmri) == 0)
5053                 return ('S');
5054         else if (strcmp(milestone->gv_name, multi_user_svr_fmri) == 0)
5055                 return ('3');
5056 
5057 #ifndef NDEBUG
5058         (void) fprintf(stderr, "%s:%d: Unknown milestone name \"%s\".\n",
5059             __FILE__, __LINE__, milestone->gv_name);
5060 #endif
5061         abort();
5062         /* NOTREACHED */
5063 }
5064 
5065 static struct {
5066         char    rl;
5067         int     sig;
5068 } init_sigs[] = {
5069         { 'S', SIGBUS },
5070         { '0', SIGINT },
5071         { '1', SIGQUIT },
5072         { '2', SIGILL },
5073         { '3', SIGTRAP },
5074         { '4', SIGIOT },
5075         { '5', SIGEMT },
5076         { '6', SIGFPE },
5077         { 0, 0 }
5078 };
5079 
5080 static void
5081 signal_init(char rl)
5082 {
5083         pid_t init_pid;
5084         int i;
5085 
5086         assert(MUTEX_HELD(&dgraph_lock));
5087 
5088         if (zone_getattr(getzoneid(), ZONE_ATTR_INITPID, &init_pid,
5089             sizeof (init_pid)) != sizeof (init_pid)) {
5090                 log_error(LOG_NOTICE, "Could not get pid to signal init.\n");
5091                 return;
5092         }
5093 
5094         for (i = 0; init_sigs[i].rl != 0; ++i)
5095                 if (init_sigs[i].rl == rl)
5096                         break;
5097 
5098         if (init_sigs[i].rl != 0) {
5099                 if (kill(init_pid, init_sigs[i].sig) != 0) {
5100                         switch (errno) {
5101                         case EPERM:
5102                         case ESRCH:
5103                                 log_error(LOG_NOTICE, "Could not signal init: "
5104                                     "%s.\n", strerror(errno));
5105                                 break;
5106 
5107                         case EINVAL:
5108                         default:
5109                                 bad_error("kill", errno);
5110                         }
5111                 }
5112         }
5113 }
5114 
5115 /*
5116  * This is called when one of the major milestones changes state, or when
5117  * init is signalled and tells us it was told to change runlevel.  We wait
5118  * to reach the milestone because this allows /etc/inittab entries to retain
5119  * some boot ordering: historically, entries could place themselves before/after
5120  * the running of /sbin/rcX scripts but we can no longer make the
5121  * distinction because the /sbin/rcX scripts no longer exist as punctuation
5122  * marks in /etc/inittab.
5123  *
5124  * Also, we only trigger an update when we reach the eventual target
5125  * milestone: without this, an /etc/inittab entry marked only for
5126  * runlevel 2 would be executed for runlevel 3, which is not how
5127  * /etc/inittab entries work.
5128  *
5129  * If we're single user coming online, then we set utmpx to the target
5130  * runlevel so that legacy scripts can work as expected.
5131  */
5132 static void
5133 graph_runlevel_changed(char rl, int online)
5134 {
5135         char trl;
5136 
5137         assert(MUTEX_HELD(&dgraph_lock));
5138 
5139         trl = target_milestone_as_runlevel();
5140 
5141         if (online) {
5142                 if (rl == trl) {
5143                         current_runlevel = trl;
5144                         signal_init(trl);
5145                 } else if (rl == 'S') {
5146                         /*
5147                          * At boot, set the entry early for the benefit of the
5148                          * legacy init scripts.
5149                          */
5150                         utmpx_set_runlevel(trl, 'S', B_FALSE);
5151                 }
5152         } else {
5153                 if (rl == '3' && trl == '2') {
5154                         current_runlevel = trl;
5155                         signal_init(trl);
5156                 } else if (rl == '2' && trl == 'S') {
5157                         current_runlevel = trl;
5158                         signal_init(trl);
5159                 }
5160         }
5161 }
5162 
5163 /*
5164  * Move to a backwards-compatible runlevel by executing the appropriate
5165  * /etc/rc?.d/K* scripts and/or setting the milestone.
5166  *
5167  * Returns
5168  *   0 - success
5169  *   ECONNRESET - success, but handle was reset
5170  *   ECONNABORTED - repository connection broken
5171  *   ECANCELED - pg was deleted
5172  */
5173 static int
5174 dgraph_set_runlevel(scf_propertygroup_t *pg, scf_property_t *prop)
5175 {
5176         char rl;
5177         scf_handle_t *h;
5178         int r;
5179         const char *ms = NULL;  /* what to commit as options/milestone */
5180         boolean_t rebound = B_FALSE;
5181         int mark_rl = 0;
5182 
5183         const char * const stop = "stop";
5184 
5185         r = libscf_extract_runlevel(prop, &rl);
5186         switch (r) {
5187         case 0:
5188                 break;
5189 
5190         case ECONNABORTED:
5191         case ECANCELED:
5192                 return (r);
5193 
5194         case EINVAL:
5195         case ENOENT:
5196                 log_error(LOG_WARNING, "runlevel property is misconfigured; "
5197                     "ignoring.\n");
5198                 /* delete the bad property */
5199                 goto nolock_out;
5200 
5201         default:
5202                 bad_error("libscf_extract_runlevel", r);
5203         }
5204 
5205         switch (rl) {
5206         case 's':
5207                 rl = 'S';
5208                 /* FALLTHROUGH */
5209 
5210         case 'S':
5211         case '2':
5212         case '3':
5213                 /*
5214                  * These cases cause a milestone change, so
5215                  * graph_runlevel_changed() will eventually deal with
5216                  * signalling init.
5217                  */
5218                 break;
5219 
5220         case '0':
5221         case '1':
5222         case '4':
5223         case '5':
5224         case '6':
5225                 mark_rl = 1;
5226                 break;
5227 
5228         default:
5229                 log_framework(LOG_NOTICE, "Unknown runlevel '%c'.\n", rl);
5230                 ms = NULL;
5231                 goto nolock_out;
5232         }
5233 
5234         h = scf_pg_handle(pg);
5235 
5236         MUTEX_LOCK(&dgraph_lock);
5237 
5238         /*
5239          * Since this triggers no milestone changes, force it by hand.
5240          */
5241         if (current_runlevel == '4' && rl == '3')
5242                 mark_rl = 1;
5243 
5244         /*
5245          * 1. If we are here after an "init X":
5246          *
5247          * init X
5248          *      init/lscf_set_runlevel()
5249          *              process_pg_event()
5250          *              dgraph_set_runlevel()
5251          *
5252          * then we haven't passed through graph_runlevel_changed() yet,
5253          * therefore 'current_runlevel' has not changed for sure but 'rl' has.
5254          * In consequence, if 'rl' is lower than 'current_runlevel', we change
5255          * the system runlevel and execute the appropriate /etc/rc?.d/K* scripts
5256          * past this test.
5257          *
5258          * 2. On the other hand, if we are here after a "svcadm milestone":
5259          *
5260          * svcadm milestone X
5261          *      dgraph_set_milestone()
5262          *              handle_graph_update_event()
5263          *              dgraph_set_instance_state()
5264          *              graph_post_X_[online|offline]()
5265          *              graph_runlevel_changed()
5266          *              signal_init()
5267          *                      init/lscf_set_runlevel()
5268          *                              process_pg_event()
5269          *                              dgraph_set_runlevel()
5270          *
5271          * then we already passed through graph_runlevel_changed() (by the way
5272          * of dgraph_set_milestone()) and 'current_runlevel' may have changed
5273          * and already be equal to 'rl' so we are going to return immediately
5274          * from dgraph_set_runlevel() without changing the system runlevel and
5275          * without executing the /etc/rc?.d/K* scripts.
5276          */
5277         if (rl == current_runlevel) {
5278                 ms = NULL;
5279                 goto out;
5280         }
5281 
5282         log_framework(LOG_DEBUG, "Changing to runlevel '%c'.\n", rl);
5283 
5284         /*
5285          * Make sure stop rc scripts see the new settings via who -r.
5286          */
5287         utmpx_set_runlevel(rl, current_runlevel, B_TRUE);
5288 
5289         /*
5290          * Some run levels don't have a direct correspondence to any
5291          * milestones, so we have to signal init directly.
5292          */
5293         if (mark_rl) {
5294                 current_runlevel = rl;
5295                 signal_init(rl);
5296         }
5297 
5298         switch (rl) {
5299         case 'S':
5300                 uu_warn("The system is coming down for administration.  "
5301                     "Please wait.\n");
5302                 fork_rc_script(rl, stop, B_FALSE);
5303                 ms = single_user_fmri;
5304                 go_single_user_mode = B_TRUE;
5305                 break;
5306 
5307         case '0':
5308                 halting_time = time(NULL);
5309                 fork_rc_script(rl, stop, B_TRUE);
5310                 halting = AD_HALT;
5311                 goto uadmin;
5312 
5313         case '5':
5314                 halting_time = time(NULL);
5315                 fork_rc_script(rl, stop, B_TRUE);
5316                 halting = AD_POWEROFF;
5317                 goto uadmin;
5318 
5319         case '6':
5320                 halting_time = time(NULL);
5321                 fork_rc_script(rl, stop, B_TRUE);
5322                 if (scf_is_fastboot_default() && getzoneid() == GLOBAL_ZONEID)
5323                         halting = AD_FASTREBOOT;
5324                 else
5325                         halting = AD_BOOT;
5326 
5327 uadmin:
5328                 uu_warn("The system is coming down.  Please wait.\n");
5329                 ms = "none";
5330 
5331                 /*
5332                  * We can't wait until all services are offline since this
5333                  * thread is responsible for taking them offline.  Instead we
5334                  * set halting to the second argument for uadmin() and call
5335                  * do_uadmin() from dgraph_set_instance_state() when
5336                  * appropriate.
5337                  */
5338                 break;
5339 
5340         case '1':
5341                 if (current_runlevel != 'S') {
5342                         uu_warn("Changing to state 1.\n");
5343                         fork_rc_script(rl, stop, B_FALSE);
5344                 } else {
5345                         uu_warn("The system is coming up for administration.  "
5346                             "Please wait.\n");
5347                 }
5348                 ms = single_user_fmri;
5349                 go_to_level1 = B_TRUE;
5350                 break;
5351 
5352         case '2':
5353                 if (current_runlevel == '3' || current_runlevel == '4')
5354                         fork_rc_script(rl, stop, B_FALSE);
5355                 ms = multi_user_fmri;
5356                 break;
5357 
5358         case '3':
5359         case '4':
5360                 ms = "all";
5361                 break;
5362 
5363         default:
5364 #ifndef NDEBUG
5365                 (void) fprintf(stderr, "%s:%d: Uncaught case %d ('%c').\n",
5366                     __FILE__, __LINE__, rl, rl);
5367 #endif
5368                 abort();
5369         }
5370 
5371 out:
5372         MUTEX_UNLOCK(&dgraph_lock);
5373 
5374 nolock_out:
5375         switch (r = libscf_clear_runlevel(pg, ms)) {
5376         case 0:
5377                 break;
5378 
5379         case ECONNABORTED:
5380                 libscf_handle_rebind(h);
5381                 rebound = B_TRUE;
5382                 goto nolock_out;
5383 
5384         case ECANCELED:
5385                 break;
5386 
5387         case EPERM:
5388         case EACCES:
5389         case EROFS:
5390                 log_error(LOG_NOTICE, "Could not delete \"%s/%s\" property: "
5391                     "%s.\n", SCF_PG_OPTIONS, "runlevel", strerror(r));
5392                 break;
5393 
5394         default:
5395                 bad_error("libscf_clear_runlevel", r);
5396         }
5397 
5398         return (rebound ? ECONNRESET : 0);
5399 }
5400 
5401 /*
5402  * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5403  * to the instances that are supposed to go offline during an
5404  * administrative disable operation.
5405  */
5406 static int
5407 mark_subtree(graph_edge_t *e, void *arg)
5408 {
5409         graph_vertex_t *v;
5410         int r;
5411 
5412         v = e->ge_vertex;
5413 
5414         /* If it's already in the subgraph, skip. */
5415         if (v->gv_flags & GV_TOOFFLINE)
5416                 return (UU_WALK_NEXT);
5417 
5418         switch (v->gv_type) {
5419         case GVT_INST:
5420                 /* If the instance is already disabled, skip it. */
5421                 if (!(v->gv_flags & GV_ENABLED))
5422                         return (UU_WALK_NEXT);
5423 
5424                 v->gv_flags |= GV_TOOFFLINE;
5425                 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5426                 break;
5427         case GVT_GROUP:
5428                 /*
5429                  * Skip all excluded dependents and decide whether to offline
5430                  * the service based on the restart_on attribute.
5431                  */
5432                 if (is_depgrp_bypassed(v))
5433                         return (UU_WALK_NEXT);
5434                 break;
5435         }
5436 
5437         r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5438             0);
5439         assert(r == 0);
5440         return (UU_WALK_NEXT);
5441 }
5442 
5443 static int
5444 mark_subgraph(graph_edge_t *e, void *arg)
5445 {
5446         graph_vertex_t *v;
5447         int r;
5448         int optional = (int)arg;
5449 
5450         v = e->ge_vertex;
5451 
5452         /* If it's already in the subgraph, skip. */
5453         if (v->gv_flags & GV_INSUBGRAPH)
5454                 return (UU_WALK_NEXT);
5455 
5456         /*
5457          * Keep track if walk has entered an optional dependency group
5458          */
5459         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_OPTIONAL_ALL) {
5460                 optional = 1;
5461         }
5462         /*
5463          * Quit if we are in an optional dependency group and the instance
5464          * is disabled
5465          */
5466         if (optional && (v->gv_type == GVT_INST) &&
5467             (!(v->gv_flags & GV_ENBLD_NOOVR)))
5468                 return (UU_WALK_NEXT);
5469 
5470         v->gv_flags |= GV_INSUBGRAPH;
5471 
5472         /* Skip all excluded dependencies. */
5473         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
5474                 return (UU_WALK_NEXT);
5475 
5476         r = uu_list_walk(v->gv_dependencies, (uu_walk_fn_t *)mark_subgraph,
5477             (void *)optional, 0);
5478         assert(r == 0);
5479         return (UU_WALK_NEXT);
5480 }
5481 
5482 /*
5483  * Bring down all services which are not dependencies of fmri.  The
5484  * dependencies of fmri (direct & indirect) will constitute the "subgraph",
5485  * and will have the GV_INSUBGRAPH flag set.  The rest must be brought down,
5486  * which means the state is "disabled", "maintenance", or "uninitialized".  We
5487  * could consider "offline" to be down, and refrain from sending start
5488  * commands for such services, but that's not strictly necessary, so we'll
5489  * decline to intrude on the state machine.  It would probably confuse users
5490  * anyway.
5491  *
5492  * The services should be brought down in reverse-dependency order, so we
5493  * can't do it all at once here.  We initiate by override-disabling the leaves
5494  * of the dependency tree -- those services which are up but have no
5495  * dependents which are up.  When they come down,
5496  * vertex_subgraph_dependencies_shutdown() will override-disable the newly
5497  * exposed leaves.  Perseverance will ensure completion.
5498  *
5499  * Sometimes we need to take action when the transition is complete, like
5500  * start sulogin or halt the system.  To tell when we're done, we initialize
5501  * non_subgraph_svcs here to be the number of services which need to come
5502  * down.  As each does, we decrement the counter.  When it hits zero, we take
5503  * the appropriate action.  See vertex_subgraph_dependencies_shutdown().
5504  *
5505  * In case we're coming up, we also remove any enable-overrides for the
5506  * services which are dependencies of fmri.
5507  *
5508  * If norepository is true, the function will not change the repository.
5509  *
5510  * The decision to change the system run level in accordance with the milestone
5511  * is taken in dgraph_set_runlevel().
5512  *
5513  * Returns
5514  *   0 - success
5515  *   ECONNRESET - success, but handle was rebound
5516  *   EINVAL - fmri is invalid (error is logged)
5517  *   EALREADY - the milestone is already set to fmri
5518  *   ENOENT - a configured vertex does not exist for fmri (an error is logged)
5519  */
5520 static int
5521 dgraph_set_milestone(const char *fmri, scf_handle_t *h, boolean_t norepository)
5522 {
5523         const char *cfmri, *fs;
5524         graph_vertex_t *nm, *v;
5525         int ret = 0, r;
5526         scf_instance_t *inst;
5527         boolean_t isall, isnone, rebound = B_FALSE;
5528 
5529         /* Validate fmri */
5530         isall = (strcmp(fmri, "all") == 0);
5531         isnone = (strcmp(fmri, "none") == 0);
5532 
5533         if (!isall && !isnone) {
5534                 if (fmri_canonify(fmri, (char **)&cfmri, B_FALSE) == EINVAL)
5535                         goto reject;
5536 
5537                 if (strcmp(cfmri, single_user_fmri) != 0 &&
5538                     strcmp(cfmri, multi_user_fmri) != 0 &&
5539                     strcmp(cfmri, multi_user_svr_fmri) != 0) {
5540                         startd_free((void *)cfmri, max_scf_fmri_size);
5541 reject:
5542                         log_framework(LOG_WARNING,
5543                             "Rejecting request for invalid milestone \"%s\".\n",
5544                             fmri);
5545                         return (EINVAL);
5546                 }
5547         }
5548 
5549         inst = safe_scf_instance_create(h);
5550 
5551         MUTEX_LOCK(&dgraph_lock);
5552 
5553         if (milestone == NULL) {
5554                 if (isall) {
5555                         log_framework(LOG_DEBUG,
5556                             "Milestone already set to all.\n");
5557                         ret = EALREADY;
5558                         goto out;
5559                 }
5560         } else if (milestone == MILESTONE_NONE) {
5561                 if (isnone) {
5562                         log_framework(LOG_DEBUG,
5563                             "Milestone already set to none.\n");
5564                         ret = EALREADY;
5565                         goto out;
5566                 }
5567         } else {
5568                 if (!isall && !isnone &&
5569                     strcmp(cfmri, milestone->gv_name) == 0) {
5570                         log_framework(LOG_DEBUG,
5571                             "Milestone already set to %s.\n", cfmri);
5572                         ret = EALREADY;
5573                         goto out;
5574                 }
5575         }
5576 
5577         if (!isall && !isnone) {
5578                 nm = vertex_get_by_name(cfmri);
5579                 if (nm == NULL || !(nm->gv_flags & GV_CONFIGURED)) {
5580                         log_framework(LOG_WARNING, "Cannot set milestone to %s "
5581                             "because no such service exists.\n", cfmri);
5582                         ret = ENOENT;
5583                         goto out;
5584                 }
5585         }
5586 
5587         log_framework(LOG_DEBUG, "Changing milestone to %s.\n", fmri);
5588 
5589         /*
5590          * Set milestone, removing the old one if this was the last reference.
5591          */
5592         if (milestone > MILESTONE_NONE)
5593                 (void) vertex_unref(milestone);
5594 
5595         if (isall)
5596                 milestone = NULL;
5597         else if (isnone)
5598                 milestone = MILESTONE_NONE;
5599         else {
5600                 milestone = nm;
5601                 /* milestone should count as a reference */
5602                 vertex_ref(milestone);
5603         }
5604 
5605         /* Clear all GV_INSUBGRAPH bits. */
5606         for (v = uu_list_first(dgraph); v != NULL; v = uu_list_next(dgraph, v))
5607                 v->gv_flags &= ~GV_INSUBGRAPH;
5608 
5609         if (!isall && !isnone) {
5610                 /* Set GV_INSUBGRAPH for milestone & descendents. */
5611                 milestone->gv_flags |= GV_INSUBGRAPH;
5612 
5613                 r = uu_list_walk(milestone->gv_dependencies,
5614                     (uu_walk_fn_t *)mark_subgraph, NULL, 0);
5615                 assert(r == 0);
5616         }
5617 
5618         /* Un-override services in the subgraph & override-disable the rest. */
5619         if (norepository)
5620                 goto out;
5621 
5622         non_subgraph_svcs = 0;
5623         for (v = uu_list_first(dgraph);
5624             v != NULL;
5625             v = uu_list_next(dgraph, v)) {
5626                 if (v->gv_type != GVT_INST ||
5627                     (v->gv_flags & GV_CONFIGURED) == 0)
5628                         continue;
5629 
5630 again:
5631                 r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
5632                     NULL, NULL, SCF_DECODE_FMRI_EXACT);
5633                 if (r != 0) {
5634                         switch (scf_error()) {
5635                         case SCF_ERROR_CONNECTION_BROKEN:
5636                         default:
5637                                 libscf_handle_rebind(h);
5638                                 rebound = B_TRUE;
5639                                 goto again;
5640 
5641                         case SCF_ERROR_NOT_FOUND:
5642                                 continue;
5643 
5644                         case SCF_ERROR_HANDLE_MISMATCH:
5645                         case SCF_ERROR_INVALID_ARGUMENT:
5646                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5647                         case SCF_ERROR_NOT_BOUND:
5648                                 bad_error("scf_handle_decode_fmri",
5649                                     scf_error());
5650                         }
5651                 }
5652 
5653                 if (isall || (v->gv_flags & GV_INSUBGRAPH)) {
5654                         r = libscf_delete_enable_ovr(inst);
5655                         fs = "libscf_delete_enable_ovr";
5656                 } else {
5657                         assert(isnone || (v->gv_flags & GV_INSUBGRAPH) == 0);
5658 
5659                         /*
5660                          * Services which are up need to come down before
5661                          * we're done, but we can only disable the leaves
5662                          * here.
5663                          */
5664 
5665                         if (up_state(v->gv_state))
5666                                 ++non_subgraph_svcs;
5667 
5668                         /* If it's already disabled, don't bother. */
5669                         if ((v->gv_flags & GV_ENABLED) == 0)
5670                                 continue;
5671 
5672                         if (!is_nonsubgraph_leaf(v))
5673                                 continue;
5674 
5675                         r = libscf_set_enable_ovr(inst, 0);
5676                         fs = "libscf_set_enable_ovr";
5677                 }
5678                 switch (r) {
5679                 case 0:
5680                 case ECANCELED:
5681                         break;
5682 
5683                 case ECONNABORTED:
5684                         libscf_handle_rebind(h);
5685                         rebound = B_TRUE;
5686                         goto again;
5687 
5688                 case EPERM:
5689                 case EROFS:
5690                         log_error(LOG_WARNING,
5691                             "Could not set %s/%s for %s: %s.\n",
5692                             SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
5693                             v->gv_name, strerror(r));
5694                         break;
5695 
5696                 default:
5697                         bad_error(fs, r);
5698                 }
5699         }
5700 
5701         if (halting != -1) {
5702                 if (non_subgraph_svcs > 1)
5703                         uu_warn("%d system services are now being stopped.\n",
5704                             non_subgraph_svcs);
5705                 else if (non_subgraph_svcs == 1)
5706                         uu_warn("One system service is now being stopped.\n");
5707                 else if (non_subgraph_svcs == 0)
5708                         do_uadmin();
5709         }
5710 
5711         ret = rebound ? ECONNRESET : 0;
5712 
5713 out:
5714         MUTEX_UNLOCK(&dgraph_lock);
5715         if (!isall && !isnone)
5716                 startd_free((void *)cfmri, max_scf_fmri_size);
5717         scf_instance_destroy(inst);
5718         return (ret);
5719 }
5720 
5721 
5722 /*
5723  * Returns 0, ECONNABORTED, or EINVAL.
5724  */
5725 static int
5726 handle_graph_update_event(scf_handle_t *h, graph_protocol_event_t *e)
5727 {
5728         int r;
5729 
5730         switch (e->gpe_type) {
5731         case GRAPH_UPDATE_RELOAD_GRAPH:
5732                 log_error(LOG_WARNING,
5733                     "graph_event: reload graph unimplemented\n");
5734                 break;
5735 
5736         case GRAPH_UPDATE_STATE_CHANGE: {
5737                 protocol_states_t *states = e->gpe_data;
5738 
5739                 switch (r = dgraph_set_instance_state(h, e->gpe_inst, states)) {
5740                 case 0:
5741                 case ENOENT:
5742                         break;
5743 
5744                 case ECONNABORTED:
5745                         return (ECONNABORTED);
5746 
5747                 case EINVAL:
5748                 default:
5749 #ifndef NDEBUG
5750                         (void) fprintf(stderr, "dgraph_set_instance_state() "
5751                             "failed with unexpected error %d at %s:%d.\n", r,
5752                             __FILE__, __LINE__);
5753 #endif
5754                         abort();
5755                 }
5756 
5757                 startd_free(states, sizeof (protocol_states_t));
5758                 break;
5759         }
5760 
5761         default:
5762                 log_error(LOG_WARNING,
5763                     "graph_event_loop received an unknown event: %d\n",
5764                     e->gpe_type);
5765                 break;
5766         }
5767 
5768         return (0);
5769 }
5770 
5771 /*
5772  * graph_event_thread()
5773  *    Wait for state changes from the restarters.
5774  */
5775 /*ARGSUSED*/
5776 void *
5777 graph_event_thread(void *unused)
5778 {
5779         scf_handle_t *h;
5780         int err;
5781 
5782         h = libscf_handle_create_bound_loop();
5783 
5784         /*CONSTCOND*/
5785         while (1) {
5786                 graph_protocol_event_t *e;
5787 
5788                 MUTEX_LOCK(&gu->gu_lock);
5789 
5790                 while (gu->gu_wakeup == 0)
5791                         (void) pthread_cond_wait(&gu->gu_cv, &gu->gu_lock);
5792 
5793                 gu->gu_wakeup = 0;
5794 
5795                 while ((e = graph_event_dequeue()) != NULL) {
5796                         MUTEX_LOCK(&e->gpe_lock);
5797                         MUTEX_UNLOCK(&gu->gu_lock);
5798 
5799                         while ((err = handle_graph_update_event(h, e)) ==
5800                             ECONNABORTED)
5801                                 libscf_handle_rebind(h);
5802 
5803                         if (err == 0)
5804                                 graph_event_release(e);
5805                         else
5806                                 graph_event_requeue(e);
5807 
5808                         MUTEX_LOCK(&gu->gu_lock);
5809                 }
5810 
5811                 MUTEX_UNLOCK(&gu->gu_lock);
5812         }
5813 
5814         /*
5815          * Unreachable for now -- there's currently no graceful cleanup
5816          * called on exit().
5817          */
5818         MUTEX_UNLOCK(&gu->gu_lock);
5819         scf_handle_destroy(h);
5820         return (NULL);
5821 }
5822 
5823 static void
5824 set_initial_milestone(scf_handle_t *h)
5825 {
5826         scf_instance_t *inst;
5827         char *fmri, *cfmri;
5828         size_t sz;
5829         int r;
5830 
5831         inst = safe_scf_instance_create(h);
5832         fmri = startd_alloc(max_scf_fmri_size);
5833 
5834         /*
5835          * If -m milestone= was specified, we want to set options_ovr/milestone
5836          * to it.  Otherwise we want to read what the milestone should be set
5837          * to.  Either way we need our inst.
5838          */
5839 get_self:
5840         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
5841             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5842                 switch (scf_error()) {
5843                 case SCF_ERROR_CONNECTION_BROKEN:
5844                         libscf_handle_rebind(h);
5845                         goto get_self;
5846 
5847                 case SCF_ERROR_NOT_FOUND:
5848                         if (st->st_subgraph != NULL &&
5849                             st->st_subgraph[0] != '\0') {
5850                                 sz = strlcpy(fmri, st->st_subgraph,
5851                                     max_scf_fmri_size);
5852                                 assert(sz < max_scf_fmri_size);
5853                         } else {
5854                                 fmri[0] = '\0';
5855                         }
5856                         break;
5857 
5858                 case SCF_ERROR_INVALID_ARGUMENT:
5859                 case SCF_ERROR_CONSTRAINT_VIOLATED:
5860                 case SCF_ERROR_HANDLE_MISMATCH:
5861                 default:
5862                         bad_error("scf_handle_decode_fmri", scf_error());
5863                 }
5864         } else {
5865                 if (st->st_subgraph != NULL && st->st_subgraph[0] != '\0') {
5866                         scf_propertygroup_t *pg;
5867 
5868                         pg = safe_scf_pg_create(h);
5869 
5870                         sz = strlcpy(fmri, st->st_subgraph, max_scf_fmri_size);
5871                         assert(sz < max_scf_fmri_size);
5872 
5873                         r = libscf_inst_get_or_add_pg(inst, SCF_PG_OPTIONS_OVR,
5874                             SCF_PG_OPTIONS_OVR_TYPE, SCF_PG_OPTIONS_OVR_FLAGS,
5875                             pg);
5876                         switch (r) {
5877                         case 0:
5878                                 break;
5879 
5880                         case ECONNABORTED:
5881                                 libscf_handle_rebind(h);
5882                                 goto get_self;
5883 
5884                         case EPERM:
5885                         case EACCES:
5886                         case EROFS:
5887                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5888                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5889                                     SCF_PROPERTY_MILESTONE, strerror(r));
5890                                 /* FALLTHROUGH */
5891 
5892                         case ECANCELED:
5893                                 sz = strlcpy(fmri, st->st_subgraph,
5894                                     max_scf_fmri_size);
5895                                 assert(sz < max_scf_fmri_size);
5896                                 break;
5897 
5898                         default:
5899                                 bad_error("libscf_inst_get_or_add_pg", r);
5900                         }
5901 
5902                         r = libscf_clear_runlevel(pg, fmri);
5903                         switch (r) {
5904                         case 0:
5905                                 break;
5906 
5907                         case ECONNABORTED:
5908                                 libscf_handle_rebind(h);
5909                                 goto get_self;
5910 
5911                         case EPERM:
5912                         case EACCES:
5913                         case EROFS:
5914                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5915                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5916                                     SCF_PROPERTY_MILESTONE, strerror(r));
5917                                 /* FALLTHROUGH */
5918 
5919                         case ECANCELED:
5920                                 sz = strlcpy(fmri, st->st_subgraph,
5921                                     max_scf_fmri_size);
5922                                 assert(sz < max_scf_fmri_size);
5923                                 break;
5924 
5925                         default:
5926                                 bad_error("libscf_clear_runlevel", r);
5927                         }
5928 
5929                         scf_pg_destroy(pg);
5930                 } else {
5931                         scf_property_t *prop;
5932                         scf_value_t *val;
5933 
5934                         prop = safe_scf_property_create(h);
5935                         val = safe_scf_value_create(h);
5936 
5937                         r = libscf_get_milestone(inst, prop, val, fmri,
5938                             max_scf_fmri_size);
5939                         switch (r) {
5940                         case 0:
5941                                 break;
5942 
5943                         case ECONNABORTED:
5944                                 libscf_handle_rebind(h);
5945                                 goto get_self;
5946 
5947                         case EINVAL:
5948                                 log_error(LOG_WARNING, "Milestone property is "
5949                                     "misconfigured.  Defaulting to \"all\".\n");
5950                                 /* FALLTHROUGH */
5951 
5952                         case ECANCELED:
5953                         case ENOENT:
5954                                 fmri[0] = '\0';
5955                                 break;
5956 
5957                         default:
5958                                 bad_error("libscf_get_milestone", r);
5959                         }
5960 
5961                         scf_value_destroy(val);
5962                         scf_property_destroy(prop);
5963                 }
5964         }
5965 
5966         if (fmri[0] == '\0' || strcmp(fmri, "all") == 0)
5967                 goto out;
5968 
5969         if (strcmp(fmri, "none") != 0) {
5970 retry:
5971                 if (scf_handle_decode_fmri(h, fmri, NULL, NULL, inst, NULL,
5972                     NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5973                         switch (scf_error()) {
5974                         case SCF_ERROR_INVALID_ARGUMENT:
5975                                 log_error(LOG_WARNING,
5976                                     "Requested milestone \"%s\" is invalid.  "
5977                                     "Reverting to \"all\".\n", fmri);
5978                                 goto out;
5979 
5980                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5981                                 log_error(LOG_WARNING, "Requested milestone "
5982                                     "\"%s\" does not specify an instance.  "
5983                                     "Reverting to \"all\".\n", fmri);
5984                                 goto out;
5985 
5986                         case SCF_ERROR_CONNECTION_BROKEN:
5987                                 libscf_handle_rebind(h);
5988                                 goto retry;
5989 
5990                         case SCF_ERROR_NOT_FOUND:
5991                                 log_error(LOG_WARNING, "Requested milestone "
5992                                     "\"%s\" not in repository.  Reverting to "
5993                                     "\"all\".\n", fmri);
5994                                 goto out;
5995 
5996                         case SCF_ERROR_HANDLE_MISMATCH:
5997                         default:
5998                                 bad_error("scf_handle_decode_fmri",
5999                                     scf_error());
6000                         }
6001                 }
6002 
6003                 r = fmri_canonify(fmri, &cfmri, B_FALSE);
6004                 assert(r == 0);
6005 
6006                 r = dgraph_add_instance(cfmri, inst, B_TRUE);
6007                 startd_free(cfmri, max_scf_fmri_size);
6008                 switch (r) {
6009                 case 0:
6010                         break;
6011 
6012                 case ECONNABORTED:
6013                         goto retry;
6014 
6015                 case EINVAL:
6016                         log_error(LOG_WARNING,
6017                             "Requested milestone \"%s\" is invalid.  "
6018                             "Reverting to \"all\".\n", fmri);
6019                         goto out;
6020 
6021                 case ECANCELED:
6022                         log_error(LOG_WARNING,
6023                             "Requested milestone \"%s\" not "
6024                             "in repository.  Reverting to \"all\".\n",
6025                             fmri);
6026                         goto out;
6027 
6028                 case EEXIST:
6029                 default:
6030                         bad_error("dgraph_add_instance", r);
6031                 }
6032         }
6033 
6034         log_console(LOG_INFO, "Booting to milestone \"%s\".\n", fmri);
6035 
6036         r = dgraph_set_milestone(fmri, h, B_FALSE);
6037         switch (r) {
6038         case 0:
6039         case ECONNRESET:
6040         case EALREADY:
6041                 break;
6042 
6043         case EINVAL:
6044         case ENOENT:
6045         default:
6046                 bad_error("dgraph_set_milestone", r);
6047         }
6048 
6049 out:
6050         startd_free(fmri, max_scf_fmri_size);
6051         scf_instance_destroy(inst);
6052 }
6053 
6054 void
6055 set_restart_milestone(scf_handle_t *h)
6056 {
6057         scf_instance_t *inst;
6058         scf_property_t *prop;
6059         scf_value_t *val;
6060         char *fmri;
6061         int r;
6062 
6063         inst = safe_scf_instance_create(h);
6064 
6065 get_self:
6066         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL,
6067             inst, NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
6068                 switch (scf_error()) {
6069                 case SCF_ERROR_CONNECTION_BROKEN:
6070                         libscf_handle_rebind(h);
6071                         goto get_self;
6072 
6073                 case SCF_ERROR_NOT_FOUND:
6074                         break;
6075 
6076                 case SCF_ERROR_INVALID_ARGUMENT:
6077                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6078                 case SCF_ERROR_HANDLE_MISMATCH:
6079                 default:
6080                         bad_error("scf_handle_decode_fmri", scf_error());
6081                 }
6082 
6083                 scf_instance_destroy(inst);
6084                 return;
6085         }
6086 
6087         prop = safe_scf_property_create(h);
6088         val = safe_scf_value_create(h);
6089         fmri = startd_alloc(max_scf_fmri_size);
6090 
6091         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6092         switch (r) {
6093         case 0:
6094                 break;
6095 
6096         case ECONNABORTED:
6097                 libscf_handle_rebind(h);
6098                 goto get_self;
6099 
6100         case ECANCELED:
6101         case ENOENT:
6102         case EINVAL:
6103                 goto out;
6104 
6105         default:
6106                 bad_error("libscf_get_milestone", r);
6107         }
6108 
6109         r = dgraph_set_milestone(fmri, h, B_TRUE);
6110         switch (r) {
6111         case 0:
6112         case ECONNRESET:
6113         case EALREADY:
6114         case EINVAL:
6115         case ENOENT:
6116                 break;
6117 
6118         default:
6119                 bad_error("dgraph_set_milestone", r);
6120         }
6121 
6122 out:
6123         startd_free(fmri, max_scf_fmri_size);
6124         scf_value_destroy(val);
6125         scf_property_destroy(prop);
6126         scf_instance_destroy(inst);
6127 }
6128 
6129 /*
6130  * void *graph_thread(void *)
6131  *
6132  * Graph management thread.
6133  */
6134 /*ARGSUSED*/
6135 void *
6136 graph_thread(void *arg)
6137 {
6138         scf_handle_t *h;
6139         int err;
6140 
6141         h = libscf_handle_create_bound_loop();
6142 
6143         if (st->st_initial)
6144                 set_initial_milestone(h);
6145 
6146         MUTEX_LOCK(&dgraph_lock);
6147         initial_milestone_set = B_TRUE;
6148         err = pthread_cond_broadcast(&initial_milestone_cv);
6149         assert(err == 0);
6150         MUTEX_UNLOCK(&dgraph_lock);
6151 
6152         libscf_populate_graph(h);
6153 
6154         if (!st->st_initial)
6155                 set_restart_milestone(h);
6156 
6157         MUTEX_LOCK(&st->st_load_lock);
6158         st->st_load_complete = 1;
6159         (void) pthread_cond_broadcast(&st->st_load_cv);
6160         MUTEX_UNLOCK(&st->st_load_lock);
6161 
6162         MUTEX_LOCK(&dgraph_lock);
6163         /*
6164          * Now that we've set st_load_complete we need to check can_come_up()
6165          * since if we booted to a milestone, then there won't be any more
6166          * state updates.
6167          */
6168         if (!go_single_user_mode && !go_to_level1 &&
6169             halting == -1) {
6170                 if (!sulogin_thread_running && !can_come_up()) {
6171                         (void) startd_thread_create(sulogin_thread, NULL);
6172                         sulogin_thread_running = B_TRUE;
6173                 }
6174         }
6175         MUTEX_UNLOCK(&dgraph_lock);
6176 
6177         (void) pthread_mutex_lock(&gu->gu_freeze_lock);
6178 
6179         /*CONSTCOND*/
6180         while (1) {
6181                 (void) pthread_cond_wait(&gu->gu_freeze_cv,
6182                     &gu->gu_freeze_lock);
6183         }
6184 
6185         /*
6186          * Unreachable for now -- there's currently no graceful cleanup
6187          * called on exit().
6188          */
6189         (void) pthread_mutex_unlock(&gu->gu_freeze_lock);
6190         scf_handle_destroy(h);
6191 
6192         return (NULL);
6193 }
6194 
6195 
6196 /*
6197  * int next_action()
6198  *   Given an array of timestamps 'a' with 'num' elements, find the
6199  *   lowest non-zero timestamp and return its index. If there are no
6200  *   non-zero elements, return -1.
6201  */
6202 static int
6203 next_action(hrtime_t *a, int num)
6204 {
6205         hrtime_t t = 0;
6206         int i = 0, smallest = -1;
6207 
6208         for (i = 0; i < num; i++) {
6209                 if (t == 0) {
6210                         t = a[i];
6211                         smallest = i;
6212                 } else if (a[i] != 0 && a[i] < t) {
6213                         t = a[i];
6214                         smallest = i;
6215                 }
6216         }
6217 
6218         if (t == 0)
6219                 return (-1);
6220         else
6221                 return (smallest);
6222 }
6223 
6224 /*
6225  * void process_actions()
6226  *   Process actions requested by the administrator. Possibilities include:
6227  *   refresh, restart, maintenance mode off, maintenance mode on,
6228  *   maintenance mode immediate, and degraded.
6229  *
6230  *   The set of pending actions is represented in the repository as a
6231  *   per-instance property group, with each action being a single property
6232  *   in that group.  This property group is converted to an array, with each
6233  *   action type having an array slot.  The actions in the array at the
6234  *   time process_actions() is called are acted on in the order of the
6235  *   timestamp (which is the value stored in the slot).  A value of zero
6236  *   indicates that there is no pending action of the type associated with
6237  *   a particular slot.
6238  *
6239  *   Sending an action event multiple times before the restarter has a
6240  *   chance to process that action will force it to be run at the last
6241  *   timestamp where it appears in the ordering.
6242  *
6243  *   Turning maintenance mode on trumps all other actions.
6244  *
6245  *   Returns 0 or ECONNABORTED.
6246  */
6247 static int
6248 process_actions(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst)
6249 {
6250         scf_property_t *prop = NULL;
6251         scf_value_t *val = NULL;
6252         scf_type_t type;
6253         graph_vertex_t *vertex;
6254         admin_action_t a;
6255         int i, ret = 0, r;
6256         hrtime_t action_ts[NACTIONS];
6257         char *inst_name;
6258 
6259         r = libscf_instance_get_fmri(inst, &inst_name);
6260         switch (r) {
6261         case 0:
6262                 break;
6263 
6264         case ECONNABORTED:
6265                 return (ECONNABORTED);
6266 
6267         case ECANCELED:
6268                 return (0);
6269 
6270         default:
6271                 bad_error("libscf_instance_get_fmri", r);
6272         }
6273 
6274         MUTEX_LOCK(&dgraph_lock);
6275 
6276         vertex = vertex_get_by_name(inst_name);
6277         if (vertex == NULL) {
6278                 MUTEX_UNLOCK(&dgraph_lock);
6279                 log_framework(LOG_DEBUG, "%s: Can't find graph vertex. "
6280                     "The instance must have been removed.\n", inst_name);
6281                 startd_free(inst_name, max_scf_fmri_size);
6282                 return (0);
6283         }
6284 
6285         prop = safe_scf_property_create(h);
6286         val = safe_scf_value_create(h);
6287 
6288         for (i = 0; i < NACTIONS; i++) {
6289                 if (scf_pg_get_property(pg, admin_actions[i], prop) != 0) {
6290                         switch (scf_error()) {
6291                         case SCF_ERROR_CONNECTION_BROKEN:
6292                         default:
6293                                 ret = ECONNABORTED;
6294                                 goto out;
6295 
6296                         case SCF_ERROR_DELETED:
6297                                 goto out;
6298 
6299                         case SCF_ERROR_NOT_FOUND:
6300                                 action_ts[i] = 0;
6301                                 continue;
6302 
6303                         case SCF_ERROR_HANDLE_MISMATCH:
6304                         case SCF_ERROR_INVALID_ARGUMENT:
6305                         case SCF_ERROR_NOT_SET:
6306                                 bad_error("scf_pg_get_property", scf_error());
6307                         }
6308                 }
6309 
6310                 if (scf_property_type(prop, &type) != 0) {
6311                         switch (scf_error()) {
6312                         case SCF_ERROR_CONNECTION_BROKEN:
6313                         default:
6314                                 ret = ECONNABORTED;
6315                                 goto out;
6316 
6317                         case SCF_ERROR_DELETED:
6318                                 action_ts[i] = 0;
6319                                 continue;
6320 
6321                         case SCF_ERROR_NOT_SET:
6322                                 bad_error("scf_property_type", scf_error());
6323                         }
6324                 }
6325 
6326                 if (type != SCF_TYPE_INTEGER) {
6327                         action_ts[i] = 0;
6328                         continue;
6329                 }
6330 
6331                 if (scf_property_get_value(prop, val) != 0) {
6332                         switch (scf_error()) {
6333                         case SCF_ERROR_CONNECTION_BROKEN:
6334                         default:
6335                                 ret = ECONNABORTED;
6336                                 goto out;
6337 
6338                         case SCF_ERROR_DELETED:
6339                                 goto out;
6340 
6341                         case SCF_ERROR_NOT_FOUND:
6342                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6343                                 action_ts[i] = 0;
6344                                 continue;
6345 
6346                         case SCF_ERROR_NOT_SET:
6347                         case SCF_ERROR_PERMISSION_DENIED:
6348                                 bad_error("scf_property_get_value",
6349                                     scf_error());
6350                         }
6351                 }
6352 
6353                 r = scf_value_get_integer(val, &action_ts[i]);
6354                 assert(r == 0);
6355         }
6356 
6357         a = ADMIN_EVENT_MAINT_ON_IMMEDIATE;
6358         if (action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ||
6359             action_ts[ADMIN_EVENT_MAINT_ON]) {
6360                 a = action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ?
6361                     ADMIN_EVENT_MAINT_ON_IMMEDIATE : ADMIN_EVENT_MAINT_ON;
6362 
6363                 vertex_send_event(vertex, admin_events[a]);
6364                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6365                 switch (r) {
6366                 case 0:
6367                 case EACCES:
6368                         break;
6369 
6370                 case ECONNABORTED:
6371                         ret = ECONNABORTED;
6372                         goto out;
6373 
6374                 case EPERM:
6375                         uu_die("Insufficient privilege.\n");
6376                         /* NOTREACHED */
6377 
6378                 default:
6379                         bad_error("libscf_unset_action", r);
6380                 }
6381         }
6382 
6383         while ((a = next_action(action_ts, NACTIONS)) != -1) {
6384                 log_framework(LOG_DEBUG,
6385                     "Graph: processing %s action for %s.\n", admin_actions[a],
6386                     inst_name);
6387 
6388                 if (a == ADMIN_EVENT_REFRESH) {
6389                         r = dgraph_refresh_instance(vertex, inst);
6390                         switch (r) {
6391                         case 0:
6392                         case ECANCELED:
6393                         case EINVAL:
6394                         case -1:
6395                                 break;
6396 
6397                         case ECONNABORTED:
6398                                 /* pg & inst are reset now, so just return. */
6399                                 ret = ECONNABORTED;
6400                                 goto out;
6401 
6402                         default:
6403                                 bad_error("dgraph_refresh_instance", r);
6404                         }
6405                 }
6406 
6407                 vertex_send_event(vertex, admin_events[a]);
6408 
6409                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6410                 switch (r) {
6411                 case 0:
6412                 case EACCES:
6413                         break;
6414 
6415                 case ECONNABORTED:
6416                         ret = ECONNABORTED;
6417                         goto out;
6418 
6419                 case EPERM:
6420                         uu_die("Insufficient privilege.\n");
6421                         /* NOTREACHED */
6422 
6423                 default:
6424                         bad_error("libscf_unset_action", r);
6425                 }
6426 
6427                 action_ts[a] = 0;
6428         }
6429 
6430 out:
6431         MUTEX_UNLOCK(&dgraph_lock);
6432 
6433         scf_property_destroy(prop);
6434         scf_value_destroy(val);
6435         startd_free(inst_name, max_scf_fmri_size);
6436         return (ret);
6437 }
6438 
6439 /*
6440  * inst and pg_name are scratch space, and are unset on entry.
6441  * Returns
6442  *   0 - success
6443  *   ECONNRESET - success, but repository handle rebound
6444  *   ECONNABORTED - repository connection broken
6445  */
6446 static int
6447 process_pg_event(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst,
6448     char *pg_name)
6449 {
6450         int r;
6451         scf_property_t *prop;
6452         scf_value_t *val;
6453         char *fmri;
6454         boolean_t rebound = B_FALSE, rebind_inst = B_FALSE;
6455 
6456         if (scf_pg_get_name(pg, pg_name, max_scf_value_size) < 0) {
6457                 switch (scf_error()) {
6458                 case SCF_ERROR_CONNECTION_BROKEN:
6459                 default:
6460                         return (ECONNABORTED);
6461 
6462                 case SCF_ERROR_DELETED:
6463                         return (0);
6464 
6465                 case SCF_ERROR_NOT_SET:
6466                         bad_error("scf_pg_get_name", scf_error());
6467                 }
6468         }
6469 
6470         if (strcmp(pg_name, SCF_PG_GENERAL) == 0 ||
6471             strcmp(pg_name, SCF_PG_GENERAL_OVR) == 0) {
6472                 r = dgraph_update_general(pg);
6473                 switch (r) {
6474                 case 0:
6475                 case ENOTSUP:
6476                 case ECANCELED:
6477                         return (0);
6478 
6479                 case ECONNABORTED:
6480                         return (ECONNABORTED);
6481 
6482                 case -1:
6483                         /* Error should have been logged. */
6484                         return (0);
6485 
6486                 default:
6487                         bad_error("dgraph_update_general", r);
6488                 }
6489         } else if (strcmp(pg_name, SCF_PG_RESTARTER_ACTIONS) == 0) {
6490                 if (scf_pg_get_parent_instance(pg, inst) != 0) {
6491                         switch (scf_error()) {
6492                         case SCF_ERROR_CONNECTION_BROKEN:
6493                                 return (ECONNABORTED);
6494 
6495                         case SCF_ERROR_DELETED:
6496                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6497                                 /* Ignore commands on services. */
6498                                 return (0);
6499 
6500                         case SCF_ERROR_NOT_BOUND:
6501                         case SCF_ERROR_HANDLE_MISMATCH:
6502                         case SCF_ERROR_NOT_SET:
6503                         default:
6504                                 bad_error("scf_pg_get_parent_instance",
6505                                     scf_error());
6506                         }
6507                 }
6508 
6509                 return (process_actions(h, pg, inst));
6510         }
6511 
6512         if (strcmp(pg_name, SCF_PG_OPTIONS) != 0 &&
6513             strcmp(pg_name, SCF_PG_OPTIONS_OVR) != 0)
6514                 return (0);
6515 
6516         /*
6517          * We only care about the options[_ovr] property groups of our own
6518          * instance, so get the fmri and compare.  Plus, once we know it's
6519          * correct, if the repository connection is broken we know exactly what
6520          * property group we were operating on, and can look it up again.
6521          */
6522         if (scf_pg_get_parent_instance(pg, inst) != 0) {
6523                 switch (scf_error()) {
6524                 case SCF_ERROR_CONNECTION_BROKEN:
6525                         return (ECONNABORTED);
6526 
6527                 case SCF_ERROR_DELETED:
6528                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6529                         return (0);
6530 
6531                 case SCF_ERROR_HANDLE_MISMATCH:
6532                 case SCF_ERROR_NOT_BOUND:
6533                 case SCF_ERROR_NOT_SET:
6534                 default:
6535                         bad_error("scf_pg_get_parent_instance",
6536                             scf_error());
6537                 }
6538         }
6539 
6540         switch (r = libscf_instance_get_fmri(inst, &fmri)) {
6541         case 0:
6542                 break;
6543 
6544         case ECONNABORTED:
6545                 return (ECONNABORTED);
6546 
6547         case ECANCELED:
6548                 return (0);
6549 
6550         default:
6551                 bad_error("libscf_instance_get_fmri", r);
6552         }
6553 
6554         if (strcmp(fmri, SCF_SERVICE_STARTD) != 0) {
6555                 startd_free(fmri, max_scf_fmri_size);
6556                 return (0);
6557         }
6558 
6559         /*
6560          * update the information events flag
6561          */
6562         if (strcmp(pg_name, SCF_PG_OPTIONS) == 0)
6563                 info_events_all = libscf_get_info_events_all(pg);
6564 
6565         prop = safe_scf_property_create(h);
6566         val = safe_scf_value_create(h);
6567 
6568         if (strcmp(pg_name, SCF_PG_OPTIONS_OVR) == 0) {
6569                 /* See if we need to set the runlevel. */
6570                 /* CONSTCOND */
6571                 if (0) {
6572 rebind_pg:
6573                         libscf_handle_rebind(h);
6574                         rebound = B_TRUE;
6575 
6576                         r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6577                         switch (r) {
6578                         case 0:
6579                                 break;
6580 
6581                         case ECONNABORTED:
6582                                 goto rebind_pg;
6583 
6584                         case ENOENT:
6585                                 goto out;
6586 
6587                         case EINVAL:
6588                         case ENOTSUP:
6589                                 bad_error("libscf_lookup_instance", r);
6590                         }
6591 
6592                         if (scf_instance_get_pg(inst, pg_name, pg) != 0) {
6593                                 switch (scf_error()) {
6594                                 case SCF_ERROR_DELETED:
6595                                 case SCF_ERROR_NOT_FOUND:
6596                                         goto out;
6597 
6598                                 case SCF_ERROR_CONNECTION_BROKEN:
6599                                         goto rebind_pg;
6600 
6601                                 case SCF_ERROR_HANDLE_MISMATCH:
6602                                 case SCF_ERROR_NOT_BOUND:
6603                                 case SCF_ERROR_NOT_SET:
6604                                 case SCF_ERROR_INVALID_ARGUMENT:
6605                                 default:
6606                                         bad_error("scf_instance_get_pg",
6607                                             scf_error());
6608                                 }
6609                         }
6610                 }
6611 
6612                 if (scf_pg_get_property(pg, "runlevel", prop) == 0) {
6613                         r = dgraph_set_runlevel(pg, prop);
6614                         switch (r) {
6615                         case ECONNRESET:
6616                                 rebound = B_TRUE;
6617                                 rebind_inst = B_TRUE;
6618                                 /* FALLTHROUGH */
6619 
6620                         case 0:
6621                                 break;
6622 
6623                         case ECONNABORTED:
6624                                 goto rebind_pg;
6625 
6626                         case ECANCELED:
6627                                 goto out;
6628 
6629                         default:
6630                                 bad_error("dgraph_set_runlevel", r);
6631                         }
6632                 } else {
6633                         switch (scf_error()) {
6634                         case SCF_ERROR_CONNECTION_BROKEN:
6635                         default:
6636                                 goto rebind_pg;
6637 
6638                         case SCF_ERROR_DELETED:
6639                                 goto out;
6640 
6641                         case SCF_ERROR_NOT_FOUND:
6642                                 break;
6643 
6644                         case SCF_ERROR_INVALID_ARGUMENT:
6645                         case SCF_ERROR_HANDLE_MISMATCH:
6646                         case SCF_ERROR_NOT_BOUND:
6647                         case SCF_ERROR_NOT_SET:
6648                                 bad_error("scf_pg_get_property", scf_error());
6649                         }
6650                 }
6651         }
6652 
6653         if (rebind_inst) {
6654 lookup_inst:
6655                 r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6656                 switch (r) {
6657                 case 0:
6658                         break;
6659 
6660                 case ECONNABORTED:
6661                         libscf_handle_rebind(h);
6662                         rebound = B_TRUE;
6663                         goto lookup_inst;
6664 
6665                 case ENOENT:
6666                         goto out;
6667 
6668                 case EINVAL:
6669                 case ENOTSUP:
6670                         bad_error("libscf_lookup_instance", r);
6671                 }
6672         }
6673 
6674         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6675         switch (r) {
6676         case 0:
6677                 break;
6678 
6679         case ECONNABORTED:
6680                 libscf_handle_rebind(h);
6681                 rebound = B_TRUE;
6682                 goto lookup_inst;
6683 
6684         case EINVAL:
6685                 log_error(LOG_NOTICE,
6686                     "%s/%s property of %s is misconfigured.\n", pg_name,
6687                     SCF_PROPERTY_MILESTONE, SCF_SERVICE_STARTD);
6688                 /* FALLTHROUGH */
6689 
6690         case ECANCELED:
6691         case ENOENT:
6692                 (void) strcpy(fmri, "all");
6693                 break;
6694 
6695         default:
6696                 bad_error("libscf_get_milestone", r);
6697         }
6698 
6699         r = dgraph_set_milestone(fmri, h, B_FALSE);
6700         switch (r) {
6701         case 0:
6702         case ECONNRESET:
6703         case EALREADY:
6704                 break;
6705 
6706         case EINVAL:
6707                 log_error(LOG_WARNING, "Milestone %s is invalid.\n", fmri);
6708                 break;
6709 
6710         case ENOENT:
6711                 log_error(LOG_WARNING, "Milestone %s does not exist.\n", fmri);
6712                 break;
6713 
6714         default:
6715                 bad_error("dgraph_set_milestone", r);
6716         }
6717 
6718 out:
6719         startd_free(fmri, max_scf_fmri_size);
6720         scf_value_destroy(val);
6721         scf_property_destroy(prop);
6722 
6723         return (rebound ? ECONNRESET : 0);
6724 }
6725 
6726 /*
6727  * process_delete() deletes an instance from the dgraph if 'fmri' is an
6728  * instance fmri or if 'fmri' matches the 'general' property group of an
6729  * instance (or the 'general/enabled' property).
6730  *
6731  * 'fmri' may be overwritten and cannot be trusted on return by the caller.
6732  */
6733 static void
6734 process_delete(char *fmri, scf_handle_t *h)
6735 {
6736         char *lfmri, *end_inst_fmri;
6737         const char *inst_name = NULL;
6738         const char *pg_name = NULL;
6739         const char *prop_name = NULL;
6740 
6741         lfmri = safe_strdup(fmri);
6742 
6743         /* Determine if the FMRI is a property group or instance */
6744         if (scf_parse_svc_fmri(lfmri, NULL, NULL, &inst_name, &pg_name,
6745             &prop_name) != SCF_SUCCESS) {
6746                 log_error(LOG_WARNING,
6747                     "Received invalid FMRI \"%s\" from repository server.\n",
6748                     fmri);
6749         } else if (inst_name != NULL && pg_name == NULL) {
6750                 (void) dgraph_remove_instance(fmri, h);
6751         } else if (inst_name != NULL && pg_name != NULL) {
6752                 /*
6753                  * If we're deleting the 'general' property group or
6754                  * 'general/enabled' property then the whole instance
6755                  * must be removed from the dgraph.
6756                  */
6757                 if (strcmp(pg_name, SCF_PG_GENERAL) != 0) {
6758                         free(lfmri);
6759                         return;
6760                 }
6761 
6762                 if (prop_name != NULL &&
6763                     strcmp(prop_name, SCF_PROPERTY_ENABLED) != 0) {
6764                         free(lfmri);
6765                         return;
6766                 }
6767 
6768                 /*
6769                  * Because the instance has already been deleted from the
6770                  * repository, we cannot use any scf_ functions to retrieve
6771                  * the instance FMRI however we can easily reconstruct it
6772                  * manually.
6773                  */
6774                 end_inst_fmri = strstr(fmri, SCF_FMRI_PROPERTYGRP_PREFIX);
6775                 if (end_inst_fmri == NULL)
6776                         bad_error("process_delete", 0);
6777 
6778                 end_inst_fmri[0] = '\0';
6779 
6780                 (void) dgraph_remove_instance(fmri, h);
6781         }
6782 
6783         free(lfmri);
6784 }
6785 
6786 /*ARGSUSED*/
6787 void *
6788 repository_event_thread(void *unused)
6789 {
6790         scf_handle_t *h;
6791         scf_propertygroup_t *pg;
6792         scf_instance_t *inst;
6793         char *fmri = startd_alloc(max_scf_fmri_size);
6794         char *pg_name = startd_alloc(max_scf_value_size);
6795         int r;
6796 
6797         h = libscf_handle_create_bound_loop();
6798 
6799         pg = safe_scf_pg_create(h);
6800         inst = safe_scf_instance_create(h);
6801 
6802 retry:
6803         if (_scf_notify_add_pgtype(h, SCF_GROUP_FRAMEWORK) != SCF_SUCCESS) {
6804                 if (scf_error() == SCF_ERROR_CONNECTION_BROKEN) {
6805                         libscf_handle_rebind(h);
6806                 } else {
6807                         log_error(LOG_WARNING,
6808                             "Couldn't set up repository notification "
6809                             "for property group type %s: %s\n",
6810                             SCF_GROUP_FRAMEWORK, scf_strerror(scf_error()));
6811 
6812                         (void) sleep(1);
6813                 }
6814 
6815                 goto retry;
6816         }
6817 
6818         /*CONSTCOND*/
6819         while (1) {
6820                 ssize_t res;
6821 
6822                 /* Note: fmri is only set on delete events. */
6823                 res = _scf_notify_wait(pg, fmri, max_scf_fmri_size);
6824                 if (res < 0) {
6825                         libscf_handle_rebind(h);
6826                         goto retry;
6827                 } else if (res == 0) {
6828                         /*
6829                          * property group modified.  inst and pg_name are
6830                          * pre-allocated scratch space.
6831                          */
6832                         if (scf_pg_update(pg) < 0) {
6833                                 switch (scf_error()) {
6834                                 case SCF_ERROR_DELETED:
6835                                         continue;
6836 
6837                                 case SCF_ERROR_CONNECTION_BROKEN:
6838                                         log_error(LOG_WARNING,
6839                                             "Lost repository event due to "
6840                                             "disconnection.\n");
6841                                         libscf_handle_rebind(h);
6842                                         goto retry;
6843 
6844                                 case SCF_ERROR_NOT_BOUND:
6845                                 case SCF_ERROR_NOT_SET:
6846                                 default:
6847                                         bad_error("scf_pg_update", scf_error());
6848                                 }
6849                         }
6850 
6851                         r = process_pg_event(h, pg, inst, pg_name);
6852                         switch (r) {
6853                         case 0:
6854                                 break;
6855 
6856                         case ECONNABORTED:
6857                                 log_error(LOG_WARNING, "Lost repository event "
6858                                     "due to disconnection.\n");
6859                                 libscf_handle_rebind(h);
6860                                 /* FALLTHROUGH */
6861 
6862                         case ECONNRESET:
6863                                 goto retry;
6864 
6865                         default:
6866                                 bad_error("process_pg_event", r);
6867                         }
6868                 } else {
6869                         /*
6870                          * Service, instance, or pg deleted.
6871                          * Don't trust fmri on return.
6872                          */
6873                         process_delete(fmri, h);
6874                 }
6875         }
6876 
6877         /*NOTREACHED*/
6878         return (NULL);
6879 }
6880 
6881 void
6882 graph_engine_start()
6883 {
6884         int err;
6885 
6886         (void) startd_thread_create(graph_thread, NULL);
6887 
6888         MUTEX_LOCK(&dgraph_lock);
6889         while (!initial_milestone_set) {
6890                 err = pthread_cond_wait(&initial_milestone_cv, &dgraph_lock);
6891                 assert(err == 0);
6892         }
6893         MUTEX_UNLOCK(&dgraph_lock);
6894 
6895         (void) startd_thread_create(repository_event_thread, NULL);
6896         (void) startd_thread_create(graph_event_thread, NULL);
6897 }