1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
  25  * Copyright 2016 RackTop Systems.
  26  */
  27 
  28 /*
  29  * graph.c - master restarter graph engine
  30  *
  31  *   The graph engine keeps a dependency graph of all service instances on the
  32  *   system, as recorded in the repository.  It decides when services should
  33  *   be brought up or down based on service states and dependencies and sends
  34  *   commands to restarters to effect any changes.  It also executes
  35  *   administrator commands sent by svcadm via the repository.
  36  *
  37  *   The graph is stored in uu_list_t *dgraph and its vertices are
  38  *   graph_vertex_t's, each of which has a name and an integer id unique to
  39  *   its name (see dict.c).  A vertex's type attribute designates the type
  40  *   of object it represents: GVT_INST for service instances, GVT_SVC for
  41  *   service objects (since service instances may depend on another service,
  42  *   rather than service instance), GVT_FILE for files (which services may
  43  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
  44  *   vertices are necessary because dependency lists may have particular
  45  *   grouping types (require any, require all, optional, or exclude) and
  46  *   event-propagation characteristics.
  47  *
  48  *   The initial graph is built by libscf_populate_graph() invoking
  49  *   dgraph_add_instance() for each instance in the repository.  The function
  50  *   adds a GVT_SVC vertex for the service if one does not already exist, adds
  51  *   a GVT_INST vertex named by the FMRI of the instance, and sets up the edges.
  52  *   The resulting web of vertices & edges associated with an instance's vertex
  53  *   includes
  54  *
  55  *     - an edge from the GVT_SVC vertex for the instance's service
  56  *
  57  *     - an edge to the GVT_INST vertex of the instance's resarter, if its
  58  *       restarter is not svc.startd
  59  *
  60  *     - edges from other GVT_INST vertices if the instance is a restarter
  61  *
  62  *     - for each dependency property group in the instance's "running"
  63  *       snapshot, an edge to a GVT_GROUP vertex named by the FMRI of the
  64  *       instance and the name of the property group
  65  *
  66  *     - for each value of the "entities" property in each dependency property
  67  *       group, an edge from the corresponding GVT_GROUP vertex to a
  68  *       GVT_INST, GVT_SVC, or GVT_FILE vertex
  69  *
  70  *     - edges from GVT_GROUP vertices for each dependent instance
  71  *
  72  *   After the edges are set up the vertex's GV_CONFIGURED flag is set.  If
  73  *   there are problems, or if a service is mentioned in a dependency but does
  74  *   not exist in the repository, the GV_CONFIGURED flag will be clear.
  75  *
  76  *   The graph and all of its vertices are protected by the dgraph_lock mutex.
  77  *   See restarter.c for more information.
  78  *
  79  *   The properties of an instance fall into two classes: immediate and
  80  *   snapshotted.  Immediate properties should have an immediate effect when
  81  *   changed.  Snapshotted properties should be read from a snapshot, so they
  82  *   only change when the snapshot changes.  The immediate properties used by
  83  *   the graph engine are general/enabled, general/restarter, and the properties
  84  *   in the restarter_actions property group.  Since they are immediate, they
  85  *   are not read out of a snapshot.  The snapshotted properties used by the
  86  *   graph engine are those in the property groups with type "dependency" and
  87  *   are read out of the "running" snapshot.  The "running" snapshot is created
  88  *   by the the graph engine as soon as possible, and it is updated, along with
  89  *   in-core copies of the data (dependency information for the graph engine) on
  90  *   receipt of the refresh command from svcadm.  In addition, the graph engine
  91  *   updates the "start" snapshot from the "running" snapshot whenever a service
  92  *   comes online.
  93  *
  94  *   When a DISABLE event is requested by the administrator, svc.startd shutdown
  95  *   the dependents first before shutting down the requested service.
  96  *   In graph_enable_by_vertex, we create a subtree that contains the dependent
  97  *   vertices by marking those vertices with the GV_TOOFFLINE flag. And we mark
  98  *   the vertex to disable with the GV_TODISABLE flag. Once the tree is created,
  99  *   we send the _ADMIN_DISABLE event to the leaves. The leaves will then
 100  *   transition from STATE_ONLINE/STATE_DEGRADED to STATE_OFFLINE/STATE_MAINT.
 101  *   In gt_enter_offline and gt_enter_maint if the vertex was in a subtree then
 102  *   we clear the GV_TOOFFLINE flag and walk the dependencies to offline the new
 103  *   exposed leaves. We do the same until we reach the last leaf (the one with
 104  *   the GV_TODISABLE flag). If the vertex to disable is also part of a larger
 105  *   subtree (eg. multiple DISABLE events on vertices in the same subtree) then
 106  *   once the first vertex is disabled (GV_TODISABLE flag is removed), we
 107  *   continue to propagate the offline event to the vertex's dependencies.
 108  *
 109  *
 110  * SMF state transition notifications
 111  *
 112  *   When an instance of a service managed by SMF changes state, svc.startd may
 113  *   publish a GPEC sysevent. All transitions to or from maintenance, a
 114  *   transition cause by a hardware error will generate an event.
 115  *   Other transitions will generate an event if there exist notification
 116  *   parameter for that transition. Notification parameters are stored in the
 117  *   SMF repository for the service/instance they refer to. System-wide
 118  *   notification parameters are stored in the global instance.
 119  *   svc.startd can be told to send events for all SMF state transitions despite
 120  *   of notification parameters by setting options/info_events_all to true in
 121  *   restarter:default
 122  *
 123  *   The set of transitions that generate events is cached in the
 124  *   dgraph_vertex_t gv_stn_tset for service/instance and in the global
 125  *   stn_global for the system-wide set. They are re-read when instances are
 126  *   refreshed.
 127  *
 128  *   The GPEC events published by svc.startd are consumed by fmd(1M). After
 129  *   processing these events, fmd(1M) publishes the processed events to
 130  *   notification agents. The notification agents read the notification
 131  *   parameters from the SMF repository through libscf(3LIB) interfaces and send
 132  *   the notification, or not, based on those parameters.
 133  *
 134  *   Subscription and publishing to the GPEC channels is done with the
 135  *   libfmevent(3LIB) wrappers fmev_[r]publish_*() and
 136  *   fmev_shdl_(un)subscribe().
 137  *
 138  */
 139 
 140 #include <sys/uadmin.h>
 141 #include <sys/wait.h>
 142 
 143 #include <assert.h>
 144 #include <errno.h>
 145 #include <fcntl.h>
 146 #include <fm/libfmevent.h>
 147 #include <libscf.h>
 148 #include <libscf_priv.h>
 149 #include <librestart.h>
 150 #include <libuutil.h>
 151 #include <locale.h>
 152 #include <poll.h>
 153 #include <pthread.h>
 154 #include <signal.h>
 155 #include <stddef.h>
 156 #include <stdio.h>
 157 #include <stdlib.h>
 158 #include <string.h>
 159 #include <strings.h>
 160 #include <sys/statvfs.h>
 161 #include <sys/uadmin.h>
 162 #include <zone.h>
 163 #if defined(__i386)
 164 #include <libgrubmgmt.h>
 165 #endif  /* __i386 */
 166 
 167 #include "startd.h"
 168 #include "protocol.h"
 169 
 170 
 171 #define MILESTONE_NONE  ((graph_vertex_t *)1)
 172 
 173 #define CONSOLE_LOGIN_FMRI      "svc:/system/console-login:default"
 174 #define FS_MINIMAL_FMRI         "svc:/system/filesystem/minimal:default"
 175 
 176 #define VERTEX_REMOVED  0       /* vertex has been freed  */
 177 #define VERTEX_INUSE    1       /* vertex is still in use */
 178 
 179 #define IS_ENABLED(v) ((v)->gv_flags & (GV_ENABLED | GV_ENBLD_NOOVR))
 180 
 181 /*
 182  * stn_global holds the tset for the system wide notification parameters.
 183  * It is updated on refresh of svc:/system/svc/global:default
 184  *
 185  * There are two assumptions that relax the need for a mutex:
 186  *     1. 32-bit value assignments are atomic
 187  *     2. Its value is consumed only in one point at
 188  *     dgraph_state_transition_notify(). There are no test and set races.
 189  *
 190  *     If either assumption is broken, we'll need a mutex to synchronize
 191  *     access to stn_global
 192  */
 193 int32_t stn_global;
 194 /*
 195  * info_events_all holds a flag to override notification parameters and send
 196  * Information events for all state transitions.
 197  * same about the need of a mutex here.
 198  */
 199 int info_events_all;
 200 
 201 /*
 202  * Services in these states are not considered 'down' by the
 203  * milestone/shutdown code.
 204  */
 205 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
 206         (state) == RESTARTER_STATE_DEGRADED || \
 207         (state) == RESTARTER_STATE_OFFLINE)
 208 
 209 #define is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
 210         ((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \
 211         (v->gv_restart < RERR_RESTART)))
 212 
 213 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
 214 static uu_list_t *dgraph;
 215 static pthread_mutex_t dgraph_lock;
 216 
 217 /*
 218  * milestone indicates the current subgraph.  When NULL, it is the entire
 219  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
 220  * services on which the target vertex depends.
 221  */
 222 static graph_vertex_t *milestone = NULL;
 223 static boolean_t initial_milestone_set = B_FALSE;
 224 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
 225 
 226 /* protected by dgraph_lock */
 227 static boolean_t sulogin_thread_running = B_FALSE;
 228 static boolean_t sulogin_running = B_FALSE;
 229 static boolean_t console_login_ready = B_FALSE;
 230 
 231 /* Number of services to come down to complete milestone transition. */
 232 static uint_t non_subgraph_svcs;
 233 
 234 /*
 235  * These variables indicate what should be done when we reach the milestone
 236  * target milestone, i.e., when non_subgraph_svcs == 0.  They are acted upon in
 237  * dgraph_set_instance_state().
 238  */
 239 static int halting = -1;
 240 static boolean_t go_single_user_mode = B_FALSE;
 241 static boolean_t go_to_level1 = B_FALSE;
 242 
 243 /*
 244  * Tracks when we started halting.
 245  */
 246 static time_t halting_time = 0;
 247 
 248 /*
 249  * This tracks the legacy runlevel to ensure we signal init and manage
 250  * utmpx entries correctly.
 251  */
 252 static char current_runlevel = '\0';
 253 
 254 /* Number of single user threads currently running */
 255 static pthread_mutex_t single_user_thread_lock;
 256 static int single_user_thread_count = 0;
 257 
 258 /* Statistics for dependency cycle-checking */
 259 static u_longlong_t dep_inserts = 0;
 260 static u_longlong_t dep_cycle_ns = 0;
 261 static u_longlong_t dep_insert_ns = 0;
 262 
 263 
 264 static const char * const emsg_invalid_restarter =
 265         "Transitioning %s to maintenance, restarter FMRI %s is invalid "
 266         "(see 'svcs -xv' for details).\n";
 267 static const char * const console_login_fmri = CONSOLE_LOGIN_FMRI;
 268 static const char * const single_user_fmri = SCF_MILESTONE_SINGLE_USER;
 269 static const char * const multi_user_fmri = SCF_MILESTONE_MULTI_USER;
 270 static const char * const multi_user_svr_fmri = SCF_MILESTONE_MULTI_USER_SERVER;
 271 
 272 
 273 /*
 274  * These services define the system being "up".  If none of them can come
 275  * online, then we will run sulogin on the console.  Note that the install ones
 276  * are for the miniroot and when installing CDs after the first.  can_come_up()
 277  * does the decision making, and an sulogin_thread() runs sulogin, which can be
 278  * started by dgraph_set_instance_state() or single_user_thread().
 279  *
 280  * NOTE: can_come_up() relies on SCF_MILESTONE_SINGLE_USER being the first
 281  * entry, which is only used when booting_to_single_user (boot -s) is set.
 282  * This is because when doing a "boot -s", sulogin is started from specials.c
 283  * after milestone/single-user comes online, for backwards compatibility.
 284  * In this case, SCF_MILESTONE_SINGLE_USER needs to be part of up_svcs
 285  * to ensure sulogin will be spawned if milestone/single-user cannot be reached.
 286  */
 287 static const char * const up_svcs[] = {
 288         SCF_MILESTONE_SINGLE_USER,
 289         CONSOLE_LOGIN_FMRI,
 290         "svc:/system/install-setup:default",
 291         "svc:/system/install:default",
 292         NULL
 293 };
 294 
 295 /* This array must have an element for each non-NULL element of up_svcs[]. */
 296 static graph_vertex_t *up_svcs_p[] = { NULL, NULL, NULL, NULL };
 297 
 298 /* These are for seed repository magic.  See can_come_up(). */
 299 static const char * const manifest_import = SCF_INSTANCE_MI;
 300 static graph_vertex_t *manifest_import_p = NULL;
 301 
 302 
 303 static char target_milestone_as_runlevel(void);
 304 static void graph_runlevel_changed(char rl, int online);
 305 static int dgraph_set_milestone(const char *, scf_handle_t *, boolean_t);
 306 static boolean_t should_be_in_subgraph(graph_vertex_t *v);
 307 static int mark_subtree(graph_edge_t *, void *);
 308 
 309 /*
 310  * graph_vertex_compare()
 311  *      This function can compare either int *id or * graph_vertex_t *gv
 312  *      values, as the vertex id is always the first element of a
 313  *      graph_vertex structure.
 314  */
 315 /* ARGSUSED */
 316 static int
 317 graph_vertex_compare(const void *lc_arg, const void *rc_arg, void *private)
 318 {
 319         int lc_id = ((const graph_vertex_t *)lc_arg)->gv_id;
 320         int rc_id = *(int *)rc_arg;
 321 
 322         if (lc_id > rc_id)
 323                 return (1);
 324         if (lc_id < rc_id)
 325                 return (-1);
 326         return (0);
 327 }
 328 
 329 void
 330 graph_init()
 331 {
 332         graph_edge_pool = startd_list_pool_create("graph_edges",
 333             sizeof (graph_edge_t), offsetof(graph_edge_t, ge_link), NULL,
 334             UU_LIST_POOL_DEBUG);
 335         assert(graph_edge_pool != NULL);
 336 
 337         graph_vertex_pool = startd_list_pool_create("graph_vertices",
 338             sizeof (graph_vertex_t), offsetof(graph_vertex_t, gv_link),
 339             graph_vertex_compare, UU_LIST_POOL_DEBUG);
 340         assert(graph_vertex_pool != NULL);
 341 
 342         (void) pthread_mutex_init(&dgraph_lock, &mutex_attrs);
 343         (void) pthread_mutex_init(&single_user_thread_lock, &mutex_attrs);
 344         dgraph = startd_list_create(graph_vertex_pool, NULL, UU_LIST_SORTED);
 345         assert(dgraph != NULL);
 346 
 347         if (!st->st_initial)
 348                 current_runlevel = utmpx_get_runlevel();
 349 
 350         log_framework(LOG_DEBUG, "Initialized graph\n");
 351 }
 352 
 353 static graph_vertex_t *
 354 vertex_get_by_name(const char *name)
 355 {
 356         int id;
 357 
 358         assert(MUTEX_HELD(&dgraph_lock));
 359 
 360         id = dict_lookup_byname(name);
 361         if (id == -1)
 362                 return (NULL);
 363 
 364         return (uu_list_find(dgraph, &id, NULL, NULL));
 365 }
 366 
 367 static graph_vertex_t *
 368 vertex_get_by_id(int id)
 369 {
 370         assert(MUTEX_HELD(&dgraph_lock));
 371 
 372         if (id == -1)
 373                 return (NULL);
 374 
 375         return (uu_list_find(dgraph, &id, NULL, NULL));
 376 }
 377 
 378 /*
 379  * Creates a new vertex with the given name, adds it to the graph, and returns
 380  * a pointer to it.  The graph lock must be held by this thread on entry.
 381  */
 382 static graph_vertex_t *
 383 graph_add_vertex(const char *name)
 384 {
 385         int id;
 386         graph_vertex_t *v;
 387         void *p;
 388         uu_list_index_t idx;
 389 
 390         assert(MUTEX_HELD(&dgraph_lock));
 391 
 392         id = dict_insert(name);
 393 
 394         v = startd_zalloc(sizeof (*v));
 395 
 396         v->gv_id = id;
 397 
 398         v->gv_name = startd_alloc(strlen(name) + 1);
 399         (void) strcpy(v->gv_name, name);
 400 
 401         v->gv_dependencies = startd_list_create(graph_edge_pool, v, 0);
 402         v->gv_dependents = startd_list_create(graph_edge_pool, v, 0);
 403 
 404         p = uu_list_find(dgraph, &id, NULL, &idx);
 405         assert(p == NULL);
 406 
 407         uu_list_node_init(v, &v->gv_link, graph_vertex_pool);
 408         uu_list_insert(dgraph, v, idx);
 409 
 410         return (v);
 411 }
 412 
 413 /*
 414  * Removes v from the graph and frees it.  The graph should be locked by this
 415  * thread, and v should have no edges associated with it.
 416  */
 417 static void
 418 graph_remove_vertex(graph_vertex_t *v)
 419 {
 420         assert(MUTEX_HELD(&dgraph_lock));
 421 
 422         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 423         assert(uu_list_numnodes(v->gv_dependents) == 0);
 424         assert(v->gv_refs == 0);
 425 
 426         startd_free(v->gv_name, strlen(v->gv_name) + 1);
 427         uu_list_destroy(v->gv_dependencies);
 428         uu_list_destroy(v->gv_dependents);
 429         uu_list_remove(dgraph, v);
 430 
 431         startd_free(v, sizeof (graph_vertex_t));
 432 }
 433 
 434 static void
 435 graph_add_edge(graph_vertex_t *fv, graph_vertex_t *tv)
 436 {
 437         graph_edge_t *e, *re;
 438         int r;
 439 
 440         assert(MUTEX_HELD(&dgraph_lock));
 441 
 442         e = startd_alloc(sizeof (graph_edge_t));
 443         re = startd_alloc(sizeof (graph_edge_t));
 444 
 445         e->ge_parent = fv;
 446         e->ge_vertex = tv;
 447 
 448         re->ge_parent = tv;
 449         re->ge_vertex = fv;
 450 
 451         uu_list_node_init(e, &e->ge_link, graph_edge_pool);
 452         r = uu_list_insert_before(fv->gv_dependencies, NULL, e);
 453         assert(r == 0);
 454 
 455         uu_list_node_init(re, &re->ge_link, graph_edge_pool);
 456         r = uu_list_insert_before(tv->gv_dependents, NULL, re);
 457         assert(r == 0);
 458 }
 459 
 460 static void
 461 graph_remove_edge(graph_vertex_t *v, graph_vertex_t *dv)
 462 {
 463         graph_edge_t *e;
 464 
 465         for (e = uu_list_first(v->gv_dependencies);
 466             e != NULL;
 467             e = uu_list_next(v->gv_dependencies, e)) {
 468                 if (e->ge_vertex == dv) {
 469                         uu_list_remove(v->gv_dependencies, e);
 470                         startd_free(e, sizeof (graph_edge_t));
 471                         break;
 472                 }
 473         }
 474 
 475         for (e = uu_list_first(dv->gv_dependents);
 476             e != NULL;
 477             e = uu_list_next(dv->gv_dependents, e)) {
 478                 if (e->ge_vertex == v) {
 479                         uu_list_remove(dv->gv_dependents, e);
 480                         startd_free(e, sizeof (graph_edge_t));
 481                         break;
 482                 }
 483         }
 484 }
 485 
 486 static void
 487 remove_inst_vertex(graph_vertex_t *v)
 488 {
 489         graph_edge_t *e;
 490         graph_vertex_t *sv;
 491         int i;
 492 
 493         assert(MUTEX_HELD(&dgraph_lock));
 494         assert(uu_list_numnodes(v->gv_dependents) == 1);
 495         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 496         assert(v->gv_refs == 0);
 497         assert((v->gv_flags & GV_CONFIGURED) == 0);
 498 
 499         e = uu_list_first(v->gv_dependents);
 500         sv = e->ge_vertex;
 501         graph_remove_edge(sv, v);
 502 
 503         for (i = 0; up_svcs[i] != NULL; ++i) {
 504                 if (up_svcs_p[i] == v)
 505                         up_svcs_p[i] = NULL;
 506         }
 507 
 508         if (manifest_import_p == v)
 509                 manifest_import_p = NULL;
 510 
 511         graph_remove_vertex(v);
 512 
 513         if (uu_list_numnodes(sv->gv_dependencies) == 0 &&
 514             uu_list_numnodes(sv->gv_dependents) == 0 &&
 515             sv->gv_refs == 0)
 516                 graph_remove_vertex(sv);
 517 }
 518 
 519 static void
 520 graph_walk_dependents(graph_vertex_t *v, void (*func)(graph_vertex_t *, void *),
 521     void *arg)
 522 {
 523         graph_edge_t *e;
 524 
 525         for (e = uu_list_first(v->gv_dependents);
 526             e != NULL;
 527             e = uu_list_next(v->gv_dependents, e))
 528                 func(e->ge_vertex, arg);
 529 }
 530 
 531 static void
 532 graph_walk_dependencies(graph_vertex_t *v, void (*func)(graph_vertex_t *,
 533         void *), void *arg)
 534 {
 535         graph_edge_t *e;
 536 
 537         assert(MUTEX_HELD(&dgraph_lock));
 538 
 539         for (e = uu_list_first(v->gv_dependencies);
 540             e != NULL;
 541             e = uu_list_next(v->gv_dependencies, e)) {
 542 
 543                 func(e->ge_vertex, arg);
 544         }
 545 }
 546 
 547 /*
 548  * Generic graph walking function.
 549  *
 550  * Given a vertex, this function will walk either dependencies
 551  * (WALK_DEPENDENCIES) or dependents (WALK_DEPENDENTS) of a vertex recursively
 552  * for the entire graph.  It will avoid cycles and never visit the same vertex
 553  * twice.
 554  *
 555  * We avoid traversing exclusion dependencies, because they are allowed to
 556  * create cycles in the graph.  When propagating satisfiability, there is no
 557  * need to walk exclusion dependencies because exclude_all_satisfied() doesn't
 558  * test for satisfiability.
 559  *
 560  * The walker takes two callbacks.  The first is called before examining the
 561  * dependents of each vertex.  The second is called on each vertex after
 562  * examining its dependents.  This allows is_path_to() to construct a path only
 563  * after the target vertex has been found.
 564  */
 565 typedef enum {
 566         WALK_DEPENDENTS,
 567         WALK_DEPENDENCIES
 568 } graph_walk_dir_t;
 569 
 570 typedef int (*graph_walk_cb_t)(graph_vertex_t *, void *);
 571 
 572 typedef struct graph_walk_info {
 573         graph_walk_dir_t        gi_dir;
 574         uchar_t                 *gi_visited;    /* vertex bitmap */
 575         int                     (*gi_pre)(graph_vertex_t *, void *);
 576         void                    (*gi_post)(graph_vertex_t *, void *);
 577         void                    *gi_arg;        /* callback arg */
 578         int                     gi_ret;         /* return value */
 579 } graph_walk_info_t;
 580 
 581 static int
 582 graph_walk_recurse(graph_edge_t *e, graph_walk_info_t *gip)
 583 {
 584         uu_list_t *list;
 585         int r;
 586         graph_vertex_t *v = e->ge_vertex;
 587         int i;
 588         uint_t b;
 589 
 590         i = v->gv_id / 8;
 591         b = 1 << (v->gv_id % 8);
 592 
 593         /*
 594          * Check to see if we've visited this vertex already.
 595          */
 596         if (gip->gi_visited[i] & b)
 597                 return (UU_WALK_NEXT);
 598 
 599         gip->gi_visited[i] |= b;
 600 
 601         /*
 602          * Don't follow exclusions.
 603          */
 604         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
 605                 return (UU_WALK_NEXT);
 606 
 607         /*
 608          * Call pre-visit callback.  If this doesn't terminate the walk,
 609          * continue search.
 610          */
 611         if ((gip->gi_ret = gip->gi_pre(v, gip->gi_arg)) == UU_WALK_NEXT) {
 612                 /*
 613                  * Recurse using appropriate list.
 614                  */
 615                 if (gip->gi_dir == WALK_DEPENDENTS)
 616                         list = v->gv_dependents;
 617                 else
 618                         list = v->gv_dependencies;
 619 
 620                 r = uu_list_walk(list, (uu_walk_fn_t *)graph_walk_recurse,
 621                     gip, 0);
 622                 assert(r == 0);
 623         }
 624 
 625         /*
 626          * Callbacks must return either UU_WALK_NEXT or UU_WALK_DONE.
 627          */
 628         assert(gip->gi_ret == UU_WALK_NEXT || gip->gi_ret == UU_WALK_DONE);
 629 
 630         /*
 631          * If given a post-callback, call the function for every vertex.
 632          */
 633         if (gip->gi_post != NULL)
 634                 (void) gip->gi_post(v, gip->gi_arg);
 635 
 636         /*
 637          * Preserve the callback's return value.  If the callback returns
 638          * UU_WALK_DONE, then we propagate that to the caller in order to
 639          * terminate the walk.
 640          */
 641         return (gip->gi_ret);
 642 }
 643 
 644 static void
 645 graph_walk(graph_vertex_t *v, graph_walk_dir_t dir,
 646     int (*pre)(graph_vertex_t *, void *),
 647     void (*post)(graph_vertex_t *, void *), void *arg)
 648 {
 649         graph_walk_info_t gi;
 650         graph_edge_t fake;
 651         size_t sz = dictionary->dict_new_id / 8 + 1;
 652 
 653         gi.gi_visited = startd_zalloc(sz);
 654         gi.gi_pre = pre;
 655         gi.gi_post = post;
 656         gi.gi_arg = arg;
 657         gi.gi_dir = dir;
 658         gi.gi_ret = 0;
 659 
 660         /*
 661          * Fake up an edge for the first iteration
 662          */
 663         fake.ge_vertex = v;
 664         (void) graph_walk_recurse(&fake, &gi);
 665 
 666         startd_free(gi.gi_visited, sz);
 667 }
 668 
 669 typedef struct child_search {
 670         int     id;             /* id of vertex to look for */
 671         uint_t  depth;          /* recursion depth */
 672         /*
 673          * While the vertex is not found, path is NULL.  After the search, if
 674          * the vertex was found then path should point to a -1-terminated
 675          * array of vertex id's which constitute the path to the vertex.
 676          */
 677         int     *path;
 678 } child_search_t;
 679 
 680 static int
 681 child_pre(graph_vertex_t *v, void *arg)
 682 {
 683         child_search_t *cs = arg;
 684 
 685         cs->depth++;
 686 
 687         if (v->gv_id == cs->id) {
 688                 cs->path = startd_alloc((cs->depth + 1) * sizeof (int));
 689                 cs->path[cs->depth] = -1;
 690                 return (UU_WALK_DONE);
 691         }
 692 
 693         return (UU_WALK_NEXT);
 694 }
 695 
 696 static void
 697 child_post(graph_vertex_t *v, void *arg)
 698 {
 699         child_search_t *cs = arg;
 700 
 701         cs->depth--;
 702 
 703         if (cs->path != NULL)
 704                 cs->path[cs->depth] = v->gv_id;
 705 }
 706 
 707 /*
 708  * Look for a path from from to to.  If one exists, returns a pointer to
 709  * a NULL-terminated array of pointers to the vertices along the path.  If
 710  * there is no path, returns NULL.
 711  */
 712 static int *
 713 is_path_to(graph_vertex_t *from, graph_vertex_t *to)
 714 {
 715         child_search_t cs;
 716 
 717         cs.id = to->gv_id;
 718         cs.depth = 0;
 719         cs.path = NULL;
 720 
 721         graph_walk(from, WALK_DEPENDENCIES, child_pre, child_post, &cs);
 722 
 723         return (cs.path);
 724 }
 725 
 726 /*
 727  * Given an array of int's as returned by is_path_to, allocates a string of
 728  * their names joined by newlines.  Returns the size of the allocated buffer
 729  * in *sz and frees path.
 730  */
 731 static void
 732 path_to_str(int *path, char **cpp, size_t *sz)
 733 {
 734         int i;
 735         graph_vertex_t *v;
 736         size_t allocd, new_allocd;
 737         char *new, *name;
 738 
 739         assert(MUTEX_HELD(&dgraph_lock));
 740         assert(path[0] != -1);
 741 
 742         allocd = 1;
 743         *cpp = startd_alloc(1);
 744         (*cpp)[0] = '\0';
 745 
 746         for (i = 0; path[i] != -1; ++i) {
 747                 name = NULL;
 748 
 749                 v = vertex_get_by_id(path[i]);
 750 
 751                 if (v == NULL)
 752                         name = "<deleted>";
 753                 else if (v->gv_type == GVT_INST || v->gv_type == GVT_SVC)
 754                         name = v->gv_name;
 755 
 756                 if (name != NULL) {
 757                         new_allocd = allocd + strlen(name) + 1;
 758                         new = startd_alloc(new_allocd);
 759                         (void) strcpy(new, *cpp);
 760                         (void) strcat(new, name);
 761                         (void) strcat(new, "\n");
 762 
 763                         startd_free(*cpp, allocd);
 764 
 765                         *cpp = new;
 766                         allocd = new_allocd;
 767                 }
 768         }
 769 
 770         startd_free(path, sizeof (int) * (i + 1));
 771 
 772         *sz = allocd;
 773 }
 774 
 775 
 776 /*
 777  * This function along with run_sulogin() implements an exclusion relationship
 778  * between system/console-login and sulogin.  run_sulogin() will fail if
 779  * system/console-login is online, and the graph engine should call
 780  * graph_clogin_start() to bring system/console-login online, which defers the
 781  * start if sulogin is running.
 782  */
 783 static void
 784 graph_clogin_start(graph_vertex_t *v)
 785 {
 786         assert(MUTEX_HELD(&dgraph_lock));
 787 
 788         if (sulogin_running)
 789                 console_login_ready = B_TRUE;
 790         else
 791                 vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 792 }
 793 
 794 static void
 795 graph_su_start(graph_vertex_t *v)
 796 {
 797         /*
 798          * /etc/inittab used to have the initial /sbin/rcS as a 'sysinit'
 799          * entry with a runlevel of 'S', before jumping to the final
 800          * target runlevel (as set in initdefault).  We mimic that legacy
 801          * behavior here.
 802          */
 803         utmpx_set_runlevel('S', '0', B_FALSE);
 804         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 805 }
 806 
 807 static void
 808 graph_post_su_online(void)
 809 {
 810         graph_runlevel_changed('S', 1);
 811 }
 812 
 813 static void
 814 graph_post_su_disable(void)
 815 {
 816         graph_runlevel_changed('S', 0);
 817 }
 818 
 819 static void
 820 graph_post_mu_online(void)
 821 {
 822         graph_runlevel_changed('2', 1);
 823 }
 824 
 825 static void
 826 graph_post_mu_disable(void)
 827 {
 828         graph_runlevel_changed('2', 0);
 829 }
 830 
 831 static void
 832 graph_post_mus_online(void)
 833 {
 834         graph_runlevel_changed('3', 1);
 835 }
 836 
 837 static void
 838 graph_post_mus_disable(void)
 839 {
 840         graph_runlevel_changed('3', 0);
 841 }
 842 
 843 static struct special_vertex_info {
 844         const char      *name;
 845         void            (*start_f)(graph_vertex_t *);
 846         void            (*post_online_f)(void);
 847         void            (*post_disable_f)(void);
 848 } special_vertices[] = {
 849         { CONSOLE_LOGIN_FMRI, graph_clogin_start, NULL, NULL },
 850         { SCF_MILESTONE_SINGLE_USER, graph_su_start,
 851             graph_post_su_online, graph_post_su_disable },
 852         { SCF_MILESTONE_MULTI_USER, NULL,
 853             graph_post_mu_online, graph_post_mu_disable },
 854         { SCF_MILESTONE_MULTI_USER_SERVER, NULL,
 855             graph_post_mus_online, graph_post_mus_disable },
 856         { NULL },
 857 };
 858 
 859 
 860 void
 861 vertex_send_event(graph_vertex_t *v, restarter_event_type_t e)
 862 {
 863         switch (e) {
 864         case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
 865                 assert(v->gv_state == RESTARTER_STATE_UNINIT);
 866 
 867                 MUTEX_LOCK(&st->st_load_lock);
 868                 st->st_load_instances++;
 869                 MUTEX_UNLOCK(&st->st_load_lock);
 870                 break;
 871 
 872         case RESTARTER_EVENT_TYPE_ENABLE:
 873                 log_framework(LOG_DEBUG, "Enabling %s.\n", v->gv_name);
 874                 assert(v->gv_state == RESTARTER_STATE_UNINIT ||
 875                     v->gv_state == RESTARTER_STATE_DISABLED ||
 876                     v->gv_state == RESTARTER_STATE_MAINT);
 877                 break;
 878 
 879         case RESTARTER_EVENT_TYPE_DISABLE:
 880         case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
 881                 log_framework(LOG_DEBUG, "Disabling %s.\n", v->gv_name);
 882                 assert(v->gv_state != RESTARTER_STATE_DISABLED);
 883                 break;
 884 
 885         case RESTARTER_EVENT_TYPE_STOP_RESET:
 886         case RESTARTER_EVENT_TYPE_STOP:
 887                 log_framework(LOG_DEBUG, "Stopping %s.\n", v->gv_name);
 888                 assert(v->gv_state == RESTARTER_STATE_DEGRADED ||
 889                     v->gv_state == RESTARTER_STATE_ONLINE);
 890                 break;
 891 
 892         case RESTARTER_EVENT_TYPE_START:
 893                 log_framework(LOG_DEBUG, "Starting %s.\n", v->gv_name);
 894                 assert(v->gv_state == RESTARTER_STATE_OFFLINE);
 895                 break;
 896 
 897         case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
 898         case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
 899         case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
 900         case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
 901         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
 902         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
 903         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
 904         case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
 905         case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
 906                 break;
 907 
 908         default:
 909 #ifndef NDEBUG
 910                 uu_warn("%s:%d: Bad event %d.\n", __FILE__, __LINE__, e);
 911 #endif
 912                 abort();
 913         }
 914 
 915         restarter_protocol_send_event(v->gv_name, v->gv_restarter_channel, e,
 916             v->gv_reason);
 917 }
 918 
 919 static void
 920 graph_unset_restarter(graph_vertex_t *v)
 921 {
 922         assert(MUTEX_HELD(&dgraph_lock));
 923         assert(v->gv_flags & GV_CONFIGURED);
 924 
 925         vertex_send_event(v, RESTARTER_EVENT_TYPE_REMOVE_INSTANCE);
 926 
 927         if (v->gv_restarter_id != -1) {
 928                 graph_vertex_t *rv;
 929 
 930                 rv = vertex_get_by_id(v->gv_restarter_id);
 931                 graph_remove_edge(v, rv);
 932         }
 933 
 934         v->gv_restarter_id = -1;
 935         v->gv_restarter_channel = NULL;
 936 }
 937 
 938 /*
 939  * Return VERTEX_REMOVED when the vertex passed in argument is deleted from the
 940  * dgraph otherwise return VERTEX_INUSE.
 941  */
 942 static int
 943 free_if_unrefed(graph_vertex_t *v)
 944 {
 945         assert(MUTEX_HELD(&dgraph_lock));
 946 
 947         if (v->gv_refs > 0)
 948                 return (VERTEX_INUSE);
 949 
 950         if (v->gv_type == GVT_SVC &&
 951             uu_list_numnodes(v->gv_dependents) == 0 &&
 952             uu_list_numnodes(v->gv_dependencies) == 0) {
 953                 graph_remove_vertex(v);
 954                 return (VERTEX_REMOVED);
 955         } else if (v->gv_type == GVT_INST &&
 956             (v->gv_flags & GV_CONFIGURED) == 0 &&
 957             uu_list_numnodes(v->gv_dependents) == 1 &&
 958             uu_list_numnodes(v->gv_dependencies) == 0) {
 959                 remove_inst_vertex(v);
 960                 return (VERTEX_REMOVED);
 961         }
 962 
 963         return (VERTEX_INUSE);
 964 }
 965 
 966 static void
 967 delete_depgroup(graph_vertex_t *v)
 968 {
 969         graph_edge_t *e;
 970         graph_vertex_t *dv;
 971 
 972         assert(MUTEX_HELD(&dgraph_lock));
 973         assert(v->gv_type == GVT_GROUP);
 974         assert(uu_list_numnodes(v->gv_dependents) == 0);
 975 
 976         while ((e = uu_list_first(v->gv_dependencies)) != NULL) {
 977                 dv = e->ge_vertex;
 978 
 979                 graph_remove_edge(v, dv);
 980 
 981                 switch (dv->gv_type) {
 982                 case GVT_INST:          /* instance dependency */
 983                 case GVT_SVC:           /* service dependency */
 984                         (void) free_if_unrefed(dv);
 985                         break;
 986 
 987                 case GVT_FILE:          /* file dependency */
 988                         assert(uu_list_numnodes(dv->gv_dependencies) == 0);
 989                         if (uu_list_numnodes(dv->gv_dependents) == 0)
 990                                 graph_remove_vertex(dv);
 991                         break;
 992 
 993                 default:
 994 #ifndef NDEBUG
 995                         uu_warn("%s:%d: Unexpected node type %d", __FILE__,
 996                             __LINE__, dv->gv_type);
 997 #endif
 998                         abort();
 999                 }
1000         }
1001 
1002         graph_remove_vertex(v);
1003 }
1004 
1005 static int
1006 delete_instance_deps_cb(graph_edge_t *e, void **ptrs)
1007 {
1008         graph_vertex_t *v = ptrs[0];
1009         boolean_t delete_restarter_dep = (boolean_t)ptrs[1];
1010         graph_vertex_t *dv;
1011 
1012         dv = e->ge_vertex;
1013 
1014         /*
1015          * We have four possibilities here:
1016          *   - GVT_INST: restarter
1017          *   - GVT_GROUP - GVT_INST: instance dependency
1018          *   - GVT_GROUP - GVT_SVC - GV_INST: service dependency
1019          *   - GVT_GROUP - GVT_FILE: file dependency
1020          */
1021         switch (dv->gv_type) {
1022         case GVT_INST:  /* restarter */
1023                 assert(dv->gv_id == v->gv_restarter_id);
1024                 if (delete_restarter_dep)
1025                         graph_remove_edge(v, dv);
1026                 break;
1027 
1028         case GVT_GROUP: /* pg dependency */
1029                 graph_remove_edge(v, dv);
1030                 delete_depgroup(dv);
1031                 break;
1032 
1033         case GVT_FILE:
1034                 /* These are currently not direct dependencies */
1035 
1036         default:
1037 #ifndef NDEBUG
1038                 uu_warn("%s:%d: Bad vertex type %d.\n", __FILE__, __LINE__,
1039                     dv->gv_type);
1040 #endif
1041                 abort();
1042         }
1043 
1044         return (UU_WALK_NEXT);
1045 }
1046 
1047 static void
1048 delete_instance_dependencies(graph_vertex_t *v, boolean_t delete_restarter_dep)
1049 {
1050         void *ptrs[2];
1051         int r;
1052 
1053         assert(MUTEX_HELD(&dgraph_lock));
1054         assert(v->gv_type == GVT_INST);
1055 
1056         ptrs[0] = v;
1057         ptrs[1] = (void *)delete_restarter_dep;
1058 
1059         r = uu_list_walk(v->gv_dependencies,
1060             (uu_walk_fn_t *)delete_instance_deps_cb, &ptrs, UU_WALK_ROBUST);
1061         assert(r == 0);
1062 }
1063 
1064 /*
1065  * int graph_insert_vertex_unconfigured()
1066  *   Insert a vertex without sending any restarter events. If the vertex
1067  *   already exists or creation is successful, return a pointer to it in *vp.
1068  *
1069  *   If type is not GVT_GROUP, dt can remain unset.
1070  *
1071  *   Returns 0, EEXIST, or EINVAL if the arguments are invalid (i.e., fmri
1072  *   doesn't agree with type, or type doesn't agree with dt).
1073  */
1074 static int
1075 graph_insert_vertex_unconfigured(const char *fmri, gv_type_t type,
1076     depgroup_type_t dt, restarter_error_t rt, graph_vertex_t **vp)
1077 {
1078         int r;
1079         int i;
1080 
1081         assert(MUTEX_HELD(&dgraph_lock));
1082 
1083         switch (type) {
1084         case GVT_SVC:
1085         case GVT_INST:
1086                 if (strncmp(fmri, "svc:", sizeof ("svc:") - 1) != 0)
1087                         return (EINVAL);
1088                 break;
1089 
1090         case GVT_FILE:
1091                 if (strncmp(fmri, "file:", sizeof ("file:") - 1) != 0)
1092                         return (EINVAL);
1093                 break;
1094 
1095         case GVT_GROUP:
1096                 if (dt <= 0 || rt < 0)
1097                         return (EINVAL);
1098                 break;
1099 
1100         default:
1101 #ifndef NDEBUG
1102                 uu_warn("%s:%d: Unknown type %d.\n", __FILE__, __LINE__, type);
1103 #endif
1104                 abort();
1105         }
1106 
1107         *vp = vertex_get_by_name(fmri);
1108         if (*vp != NULL)
1109                 return (EEXIST);
1110 
1111         *vp = graph_add_vertex(fmri);
1112 
1113         (*vp)->gv_type = type;
1114         (*vp)->gv_depgroup = dt;
1115         (*vp)->gv_restart = rt;
1116 
1117         (*vp)->gv_flags = 0;
1118         (*vp)->gv_state = RESTARTER_STATE_NONE;
1119 
1120         for (i = 0; special_vertices[i].name != NULL; ++i) {
1121                 if (strcmp(fmri, special_vertices[i].name) == 0) {
1122                         (*vp)->gv_start_f = special_vertices[i].start_f;
1123                         (*vp)->gv_post_online_f =
1124                             special_vertices[i].post_online_f;
1125                         (*vp)->gv_post_disable_f =
1126                             special_vertices[i].post_disable_f;
1127                         break;
1128                 }
1129         }
1130 
1131         (*vp)->gv_restarter_id = -1;
1132         (*vp)->gv_restarter_channel = 0;
1133 
1134         if (type == GVT_INST) {
1135                 char *sfmri;
1136                 graph_vertex_t *sv;
1137 
1138                 sfmri = inst_fmri_to_svc_fmri(fmri);
1139                 sv = vertex_get_by_name(sfmri);
1140                 if (sv == NULL) {
1141                         r = graph_insert_vertex_unconfigured(sfmri, GVT_SVC, 0,
1142                             0, &sv);
1143                         assert(r == 0);
1144                 }
1145                 startd_free(sfmri, max_scf_fmri_size);
1146 
1147                 graph_add_edge(sv, *vp);
1148         }
1149 
1150         /*
1151          * If this vertex is in the subgraph, mark it as so, for both
1152          * GVT_INST and GVT_SERVICE verteces.
1153          * A GVT_SERVICE vertex can only be in the subgraph if another instance
1154          * depends on it, in which case it's already been added to the graph
1155          * and marked as in the subgraph (by refresh_vertex()).  If a
1156          * GVT_SERVICE vertex was freshly added (by the code above), it means
1157          * that it has no dependents, and cannot be in the subgraph.
1158          * Regardless of this, we still check that gv_flags includes
1159          * GV_INSUBGRAPH in the event that future behavior causes the above
1160          * code to add a GVT_SERVICE vertex which should be in the subgraph.
1161          */
1162 
1163         (*vp)->gv_flags |= (should_be_in_subgraph(*vp)? GV_INSUBGRAPH : 0);
1164 
1165         return (0);
1166 }
1167 
1168 /*
1169  * Returns 0 on success or ELOOP if the dependency would create a cycle.
1170  */
1171 static int
1172 graph_insert_dependency(graph_vertex_t *fv, graph_vertex_t *tv, int **pathp)
1173 {
1174         hrtime_t now;
1175 
1176         assert(MUTEX_HELD(&dgraph_lock));
1177 
1178         /* cycle detection */
1179         now = gethrtime();
1180 
1181         /* Don't follow exclusions. */
1182         if (!(fv->gv_type == GVT_GROUP &&
1183             fv->gv_depgroup == DEPGRP_EXCLUDE_ALL)) {
1184                 *pathp = is_path_to(tv, fv);
1185                 if (*pathp)
1186                         return (ELOOP);
1187         }
1188 
1189         dep_cycle_ns += gethrtime() - now;
1190         ++dep_inserts;
1191         now = gethrtime();
1192 
1193         graph_add_edge(fv, tv);
1194 
1195         dep_insert_ns += gethrtime() - now;
1196 
1197         /* Check if the dependency adds the "to" vertex to the subgraph */
1198         tv->gv_flags |= (should_be_in_subgraph(tv) ? GV_INSUBGRAPH : 0);
1199 
1200         return (0);
1201 }
1202 
1203 static int
1204 inst_running(graph_vertex_t *v)
1205 {
1206         assert(v->gv_type == GVT_INST);
1207 
1208         if (v->gv_state == RESTARTER_STATE_ONLINE ||
1209             v->gv_state == RESTARTER_STATE_DEGRADED)
1210                 return (1);
1211 
1212         return (0);
1213 }
1214 
1215 /*
1216  * The dependency evaluation functions return
1217  *   1 - dependency satisfied
1218  *   0 - dependency unsatisfied
1219  *   -1 - dependency unsatisfiable (without administrator intervention)
1220  *
1221  * The functions also take a boolean satbility argument.  When true, the
1222  * functions may recurse in order to determine satisfiability.
1223  */
1224 static int require_any_satisfied(graph_vertex_t *, boolean_t);
1225 static int dependency_satisfied(graph_vertex_t *, boolean_t);
1226 
1227 /*
1228  * A require_all dependency is unsatisfied if any elements are unsatisfied.  It
1229  * is unsatisfiable if any elements are unsatisfiable.
1230  */
1231 static int
1232 require_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1233 {
1234         graph_edge_t *edge;
1235         int i;
1236         boolean_t any_unsatisfied;
1237 
1238         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1239                 return (1);
1240 
1241         any_unsatisfied = B_FALSE;
1242 
1243         for (edge = uu_list_first(groupv->gv_dependencies);
1244             edge != NULL;
1245             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1246                 i = dependency_satisfied(edge->ge_vertex, satbility);
1247                 if (i == 1)
1248                         continue;
1249 
1250                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1251                     "require_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1252                     edge->ge_vertex->gv_name, i == 0 ? "ed" : "able");
1253 
1254                 if (!satbility)
1255                         return (0);
1256 
1257                 if (i == -1)
1258                         return (-1);
1259 
1260                 any_unsatisfied = B_TRUE;
1261         }
1262 
1263         return (any_unsatisfied ? 0 : 1);
1264 }
1265 
1266 /*
1267  * A require_any dependency is satisfied if any element is satisfied.  It is
1268  * satisfiable if any element is satisfiable.
1269  */
1270 static int
1271 require_any_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1272 {
1273         graph_edge_t *edge;
1274         int s;
1275         boolean_t satisfiable;
1276 
1277         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1278                 return (1);
1279 
1280         satisfiable = B_FALSE;
1281 
1282         for (edge = uu_list_first(groupv->gv_dependencies);
1283             edge != NULL;
1284             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1285                 s = dependency_satisfied(edge->ge_vertex, satbility);
1286 
1287                 if (s == 1)
1288                         return (1);
1289 
1290                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1291                     "require_any(%s): %s is unsatisfi%s.\n",
1292                     groupv->gv_name, edge->ge_vertex->gv_name,
1293                     s == 0 ? "ed" : "able");
1294 
1295                 if (satbility && s == 0)
1296                         satisfiable = B_TRUE;
1297         }
1298 
1299         return (!satbility || satisfiable ? 0 : -1);
1300 }
1301 
1302 /*
1303  * An optional_all dependency only considers elements which are configured,
1304  * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1305  * is unsatisfied.
1306  *
1307  * Offline dependencies which are waiting for a dependency to come online are
1308  * unsatisfied.  Offline dependences which cannot possibly come online
1309  * (unsatisfiable) are always considered satisfied.
1310  */
1311 static int
1312 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1313 {
1314         graph_edge_t *edge;
1315         graph_vertex_t *v;
1316         boolean_t any_qualified;
1317         boolean_t any_unsatisfied;
1318         int i;
1319 
1320         any_qualified = B_FALSE;
1321         any_unsatisfied = B_FALSE;
1322 
1323         for (edge = uu_list_first(groupv->gv_dependencies);
1324             edge != NULL;
1325             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1326                 v = edge->ge_vertex;
1327 
1328                 switch (v->gv_type) {
1329                 case GVT_INST:
1330                         /* Skip missing instances */
1331                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1332                                 continue;
1333 
1334                         if (v->gv_state == RESTARTER_STATE_MAINT)
1335                                 continue;
1336 
1337                         any_qualified = B_TRUE;
1338                         if (v->gv_state == RESTARTER_STATE_OFFLINE) {
1339                                 /*
1340                                  * For offline dependencies, treat unsatisfiable
1341                                  * as satisfied.
1342                                  */
1343                                 i = dependency_satisfied(v, B_TRUE);
1344                                 if (i == -1)
1345                                         i = 1;
1346                         } else if (v->gv_state == RESTARTER_STATE_DISABLED) {
1347                                 /*
1348                                  * The service is enabled, but hasn't
1349                                  * transitioned out of disabled yet.  Treat it
1350                                  * as unsatisfied (not unsatisfiable).
1351                                  */
1352                                 i = v->gv_flags & GV_ENABLED ? 0 : 1;
1353                         } else {
1354                                 i = dependency_satisfied(v, satbility);
1355                         }
1356                         break;
1357 
1358                 case GVT_FILE:
1359                         any_qualified = B_TRUE;
1360                         i = dependency_satisfied(v, satbility);
1361 
1362                         break;
1363 
1364                 case GVT_SVC: {
1365                         boolean_t svc_any_qualified;
1366                         boolean_t svc_satisfied;
1367                         boolean_t svc_satisfiable;
1368                         graph_vertex_t *v2;
1369                         graph_edge_t *e2;
1370 
1371                         svc_any_qualified = B_FALSE;
1372                         svc_satisfied = B_FALSE;
1373                         svc_satisfiable = B_FALSE;
1374 
1375                         for (e2 = uu_list_first(v->gv_dependencies);
1376                             e2 != NULL;
1377                             e2 = uu_list_next(v->gv_dependencies, e2)) {
1378                                 v2 = e2->ge_vertex;
1379                                 assert(v2->gv_type == GVT_INST);
1380 
1381                                 if ((v2->gv_flags & GV_CONFIGURED) == 0)
1382                                         continue;
1383 
1384                                 if (v2->gv_state == RESTARTER_STATE_MAINT)
1385                                         continue;
1386 
1387                                 svc_any_qualified = B_TRUE;
1388                                 if (v2->gv_state == RESTARTER_STATE_OFFLINE) {
1389                                         /*
1390                                          * For offline dependencies, treat
1391                                          * unsatisfiable as satisfied.
1392                                          */
1393                                         i = dependency_satisfied(v2, B_TRUE);
1394                                         if (i == -1)
1395                                                 i = 1;
1396                                 } else if (v2->gv_state ==
1397                                     RESTARTER_STATE_DISABLED) {
1398                                         i = v2->gv_flags & GV_ENABLED ? 0 : 1;
1399                                 } else {
1400                                         i = dependency_satisfied(v2, satbility);
1401                                 }
1402 
1403                                 if (i == 1) {
1404                                         svc_satisfied = B_TRUE;
1405                                         break;
1406                                 }
1407                                 if (i == 0)
1408                                         svc_satisfiable = B_TRUE;
1409                         }
1410 
1411                         if (!svc_any_qualified)
1412                                 continue;
1413                         any_qualified = B_TRUE;
1414                         if (svc_satisfied) {
1415                                 i = 1;
1416                         } else if (svc_satisfiable) {
1417                                 i = 0;
1418                         } else {
1419                                 i = -1;
1420                         }
1421                         break;
1422                 }
1423 
1424                 case GVT_GROUP:
1425                 default:
1426 #ifndef NDEBUG
1427                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1428                             __LINE__, v->gv_type);
1429 #endif
1430                         abort();
1431                 }
1432 
1433                 if (i == 1)
1434                         continue;
1435 
1436                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1437                     "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1438                     v->gv_name, i == 0 ? "ed" : "able");
1439 
1440                 if (!satbility)
1441                         return (0);
1442                 if (i == -1)
1443                         return (-1);
1444                 any_unsatisfied = B_TRUE;
1445         }
1446 
1447         if (!any_qualified)
1448                 return (1);
1449 
1450         return (any_unsatisfied ? 0 : 1);
1451 }
1452 
1453 /*
1454  * An exclude_all dependency is unsatisfied if any non-service element is
1455  * satisfied or any service instance which is configured, enabled, and not in
1456  * maintenance is satisfied.  Usually when unsatisfied, it is also
1457  * unsatisfiable.
1458  */
1459 #define LOG_EXCLUDE(u, v)                                               \
1460         log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,                   \
1461             "exclude_all(%s): %s is satisfied.\n",                      \
1462             (u)->gv_name, (v)->gv_name)
1463 
1464 /* ARGSUSED */
1465 static int
1466 exclude_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1467 {
1468         graph_edge_t *edge, *e2;
1469         graph_vertex_t *v, *v2;
1470 
1471         for (edge = uu_list_first(groupv->gv_dependencies);
1472             edge != NULL;
1473             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1474                 v = edge->ge_vertex;
1475 
1476                 switch (v->gv_type) {
1477                 case GVT_INST:
1478                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1479                                 continue;
1480 
1481                         switch (v->gv_state) {
1482                         case RESTARTER_STATE_ONLINE:
1483                         case RESTARTER_STATE_DEGRADED:
1484                                 LOG_EXCLUDE(groupv, v);
1485                                 return (v->gv_flags & GV_ENABLED ? -1 : 0);
1486 
1487                         case RESTARTER_STATE_OFFLINE:
1488                         case RESTARTER_STATE_UNINIT:
1489                                 LOG_EXCLUDE(groupv, v);
1490                                 return (0);
1491 
1492                         case RESTARTER_STATE_DISABLED:
1493                         case RESTARTER_STATE_MAINT:
1494                                 continue;
1495 
1496                         default:
1497 #ifndef NDEBUG
1498                                 uu_warn("%s:%d: Unexpected vertex state %d.\n",
1499                                     __FILE__, __LINE__, v->gv_state);
1500 #endif
1501                                 abort();
1502                         }
1503                         /* NOTREACHED */
1504 
1505                 case GVT_SVC:
1506                         break;
1507 
1508                 case GVT_FILE:
1509                         if (!file_ready(v))
1510                                 continue;
1511                         LOG_EXCLUDE(groupv, v);
1512                         return (-1);
1513 
1514                 case GVT_GROUP:
1515                 default:
1516 #ifndef NDEBUG
1517                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1518                             __LINE__, v->gv_type);
1519 #endif
1520                         abort();
1521                 }
1522 
1523                 /* v represents a service */
1524                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1525                         continue;
1526 
1527                 for (e2 = uu_list_first(v->gv_dependencies);
1528                     e2 != NULL;
1529                     e2 = uu_list_next(v->gv_dependencies, e2)) {
1530                         v2 = e2->ge_vertex;
1531                         assert(v2->gv_type == GVT_INST);
1532 
1533                         if ((v2->gv_flags & GV_CONFIGURED) == 0)
1534                                 continue;
1535 
1536                         switch (v2->gv_state) {
1537                         case RESTARTER_STATE_ONLINE:
1538                         case RESTARTER_STATE_DEGRADED:
1539                                 LOG_EXCLUDE(groupv, v2);
1540                                 return (v2->gv_flags & GV_ENABLED ? -1 : 0);
1541 
1542                         case RESTARTER_STATE_OFFLINE:
1543                         case RESTARTER_STATE_UNINIT:
1544                                 LOG_EXCLUDE(groupv, v2);
1545                                 return (0);
1546 
1547                         case RESTARTER_STATE_DISABLED:
1548                         case RESTARTER_STATE_MAINT:
1549                                 continue;
1550 
1551                         default:
1552 #ifndef NDEBUG
1553                                 uu_warn("%s:%d: Unexpected vertex type %d.\n",
1554                                     __FILE__, __LINE__, v2->gv_type);
1555 #endif
1556                                 abort();
1557                         }
1558                 }
1559         }
1560 
1561         return (1);
1562 }
1563 
1564 /*
1565  * int instance_satisfied()
1566  *   Determine if all the dependencies are satisfied for the supplied instance
1567  *   vertex. Return 1 if they are, 0 if they aren't, and -1 if they won't be
1568  *   without administrator intervention.
1569  */
1570 static int
1571 instance_satisfied(graph_vertex_t *v, boolean_t satbility)
1572 {
1573         assert(v->gv_type == GVT_INST);
1574         assert(!inst_running(v));
1575 
1576         return (require_all_satisfied(v, satbility));
1577 }
1578 
1579 /*
1580  * Decide whether v can satisfy a dependency.  v can either be a child of
1581  * a group vertex, or of an instance vertex.
1582  */
1583 static int
1584 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1585 {
1586         switch (v->gv_type) {
1587         case GVT_INST:
1588                 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1589                         if (v->gv_flags & GV_DEATHROW) {
1590                                 /*
1591                                  * A dependency on an instance with GV_DEATHROW
1592                                  * flag is always considered as satisfied.
1593                                  */
1594                                 return (1);
1595                         }
1596                         return (-1);
1597                 }
1598 
1599                 /*
1600                  * Any vertex with the GV_TOOFFLINE flag set is guaranteed
1601                  * to have its dependencies unsatisfiable.
1602                  */
1603                 if (v->gv_flags & GV_TOOFFLINE)
1604                         return (-1);
1605 
1606                 switch (v->gv_state) {
1607                 case RESTARTER_STATE_ONLINE:
1608                 case RESTARTER_STATE_DEGRADED:
1609                         return (1);
1610 
1611                 case RESTARTER_STATE_OFFLINE:
1612                         if (!satbility)
1613                                 return (0);
1614                         return (instance_satisfied(v, satbility) != -1 ?
1615                             0 : -1);
1616 
1617                 case RESTARTER_STATE_DISABLED:
1618                 case RESTARTER_STATE_MAINT:
1619                         return (-1);
1620 
1621                 case RESTARTER_STATE_UNINIT:
1622                         return (0);
1623 
1624                 default:
1625 #ifndef NDEBUG
1626                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
1627                             __FILE__, __LINE__, v->gv_state);
1628 #endif
1629                         abort();
1630                         /* NOTREACHED */
1631                 }
1632 
1633         case GVT_SVC:
1634                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1635                         return (-1);
1636                 return (require_any_satisfied(v, satbility));
1637 
1638         case GVT_FILE:
1639                 /* i.e., we assume files will not be automatically generated */
1640                 return (file_ready(v) ? 1 : -1);
1641 
1642         case GVT_GROUP:
1643                 break;
1644 
1645         default:
1646 #ifndef NDEBUG
1647                 uu_warn("%s:%d: Unexpected node type %d.\n", __FILE__, __LINE__,
1648                     v->gv_type);
1649 #endif
1650                 abort();
1651                 /* NOTREACHED */
1652         }
1653 
1654         switch (v->gv_depgroup) {
1655         case DEPGRP_REQUIRE_ANY:
1656                 return (require_any_satisfied(v, satbility));
1657 
1658         case DEPGRP_REQUIRE_ALL:
1659                 return (require_all_satisfied(v, satbility));
1660 
1661         case DEPGRP_OPTIONAL_ALL:
1662                 return (optional_all_satisfied(v, satbility));
1663 
1664         case DEPGRP_EXCLUDE_ALL:
1665                 return (exclude_all_satisfied(v, satbility));
1666 
1667         default:
1668 #ifndef NDEBUG
1669                 uu_warn("%s:%d: Unknown dependency grouping %d.\n", __FILE__,
1670                     __LINE__, v->gv_depgroup);
1671 #endif
1672                 abort();
1673         }
1674 }
1675 
1676 void
1677 graph_start_if_satisfied(graph_vertex_t *v)
1678 {
1679         if (v->gv_state == RESTARTER_STATE_OFFLINE &&
1680             instance_satisfied(v, B_FALSE) == 1) {
1681                 if (v->gv_start_f == NULL)
1682                         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
1683                 else
1684                         v->gv_start_f(v);
1685         }
1686 }
1687 
1688 /*
1689  * propagate_satbility()
1690  *
1691  * This function is used when the given vertex changes state in such a way that
1692  * one of its dependents may become unsatisfiable.  This happens when an
1693  * instance transitions between offline -> online, or from !running ->
1694  * maintenance, as well as when an instance is removed from the graph.
1695  *
1696  * We have to walk all the dependents, since optional_all dependencies several
1697  * levels up could become (un)satisfied, instead of unsatisfiable.  For example,
1698  *
1699  *      +-----+  optional_all  +-----+  require_all  +-----+
1700  *      |  A  |--------------->|  B  |-------------->|  C  |
1701  *      +-----+                +-----+               +-----+
1702  *
1703  *                                              offline -> maintenance
1704  *
1705  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1706  * an optional dependency, what was previously an unsatisfiable situation is now
1707  * satisfied (B will never come online, even though its state hasn't changed).
1708  *
1709  * Note that it's not necessary to continue examining dependents after reaching
1710  * an optional_all dependency.  It's not possible for an optional_all dependency
1711  * to change satisfiability without also coming online, in which case we get a
1712  * start event and propagation continues naturally.  However, it does no harm to
1713  * continue propagating satisfiability (as it is a relatively rare event), and
1714  * keeps the walker code simple and generic.
1715  */
1716 /*ARGSUSED*/
1717 static int
1718 satbility_cb(graph_vertex_t *v, void *arg)
1719 {
1720         if (v->gv_type == GVT_INST)
1721                 graph_start_if_satisfied(v);
1722 
1723         return (UU_WALK_NEXT);
1724 }
1725 
1726 static void
1727 propagate_satbility(graph_vertex_t *v)
1728 {
1729         graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1730 }
1731 
1732 static void propagate_stop(graph_vertex_t *, void *);
1733 
1734 /* ARGSUSED */
1735 static void
1736 propagate_start(graph_vertex_t *v, void *arg)
1737 {
1738         switch (v->gv_type) {
1739         case GVT_INST:
1740                 graph_start_if_satisfied(v);
1741                 break;
1742 
1743         case GVT_GROUP:
1744                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1745                         graph_walk_dependents(v, propagate_stop,
1746                             (void *)RERR_RESTART);
1747                         break;
1748                 }
1749                 /* FALLTHROUGH */
1750 
1751         case GVT_SVC:
1752                 graph_walk_dependents(v, propagate_start, NULL);
1753                 break;
1754 
1755         case GVT_FILE:
1756 #ifndef NDEBUG
1757                 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1758                     __FILE__, __LINE__);
1759 #endif
1760                 abort();
1761                 /* NOTREACHED */
1762 
1763         default:
1764 #ifndef NDEBUG
1765                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1766                     v->gv_type);
1767 #endif
1768                 abort();
1769         }
1770 }
1771 
1772 static void
1773 propagate_stop(graph_vertex_t *v, void *arg)
1774 {
1775         graph_edge_t *e;
1776         graph_vertex_t *svc;
1777         restarter_error_t err = (restarter_error_t)arg;
1778 
1779         switch (v->gv_type) {
1780         case GVT_INST:
1781                 /* Restarter */
1782                 if (err > RERR_NONE && inst_running(v)) {
1783                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1784                                 vertex_send_event(v,
1785                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1786                         } else {
1787                                 vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1788                         }
1789                 }
1790                 break;
1791 
1792         case GVT_SVC:
1793                 graph_walk_dependents(v, propagate_stop, arg);
1794                 break;
1795 
1796         case GVT_FILE:
1797 #ifndef NDEBUG
1798                 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1799                     __FILE__, __LINE__);
1800 #endif
1801                 abort();
1802                 /* NOTREACHED */
1803 
1804         case GVT_GROUP:
1805                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1806                         graph_walk_dependents(v, propagate_start, NULL);
1807                         break;
1808                 }
1809 
1810                 if (err == RERR_NONE || err > v->gv_restart)
1811                         break;
1812 
1813                 assert(uu_list_numnodes(v->gv_dependents) == 1);
1814                 e = uu_list_first(v->gv_dependents);
1815                 svc = e->ge_vertex;
1816 
1817                 if (inst_running(svc)) {
1818                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1819                                 vertex_send_event(svc,
1820                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1821                         } else {
1822                                 vertex_send_event(svc,
1823                                     RESTARTER_EVENT_TYPE_STOP);
1824                         }
1825                 }
1826                 break;
1827 
1828         default:
1829 #ifndef NDEBUG
1830                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1831                     v->gv_type);
1832 #endif
1833                 abort();
1834         }
1835 }
1836 
1837 void
1838 offline_vertex(graph_vertex_t *v)
1839 {
1840         scf_handle_t *h = libscf_handle_create_bound_loop();
1841         scf_instance_t *scf_inst = safe_scf_instance_create(h);
1842         scf_propertygroup_t *pg = safe_scf_pg_create(h);
1843         restarter_instance_state_t state, next_state;
1844         int r;
1845 
1846         assert(v->gv_type == GVT_INST);
1847 
1848         if (scf_inst == NULL)
1849                 bad_error("safe_scf_instance_create", scf_error());
1850         if (pg == NULL)
1851                 bad_error("safe_scf_pg_create", scf_error());
1852 
1853         /* if the vertex is already going offline, return */
1854 rep_retry:
1855         if (scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, scf_inst, NULL,
1856             NULL, SCF_DECODE_FMRI_EXACT) != 0) {
1857                 switch (scf_error()) {
1858                 case SCF_ERROR_CONNECTION_BROKEN:
1859                         libscf_handle_rebind(h);
1860                         goto rep_retry;
1861 
1862                 case SCF_ERROR_NOT_FOUND:
1863                         scf_pg_destroy(pg);
1864                         scf_instance_destroy(scf_inst);
1865                         (void) scf_handle_unbind(h);
1866                         scf_handle_destroy(h);
1867                         return;
1868                 }
1869                 uu_die("Can't decode FMRI %s: %s\n", v->gv_name,
1870                     scf_strerror(scf_error()));
1871         }
1872 
1873         r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
1874         if (r != 0) {
1875                 switch (scf_error()) {
1876                 case SCF_ERROR_CONNECTION_BROKEN:
1877                         libscf_handle_rebind(h);
1878                         goto rep_retry;
1879 
1880                 case SCF_ERROR_NOT_SET:
1881                 case SCF_ERROR_NOT_FOUND:
1882                         scf_pg_destroy(pg);
1883                         scf_instance_destroy(scf_inst);
1884                         (void) scf_handle_unbind(h);
1885                         scf_handle_destroy(h);
1886                         return;
1887 
1888                 default:
1889                         bad_error("scf_instance_get_pg", scf_error());
1890                 }
1891         } else {
1892                 r = libscf_read_states(pg, &state, &next_state);
1893                 if (r == 0 && (next_state == RESTARTER_STATE_OFFLINE ||
1894                     next_state == RESTARTER_STATE_DISABLED)) {
1895                         log_framework(LOG_DEBUG,
1896                             "%s: instance is already going down.\n",
1897                             v->gv_name);
1898                         scf_pg_destroy(pg);
1899                         scf_instance_destroy(scf_inst);
1900                         (void) scf_handle_unbind(h);
1901                         scf_handle_destroy(h);
1902                         return;
1903                 }
1904         }
1905 
1906         scf_pg_destroy(pg);
1907         scf_instance_destroy(scf_inst);
1908         (void) scf_handle_unbind(h);
1909         scf_handle_destroy(h);
1910 
1911         vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP_RESET);
1912 }
1913 
1914 /*
1915  * void graph_enable_by_vertex()
1916  *   If admin is non-zero, this is an administrative request for change
1917  *   of the enabled property.  Thus, send the ADMIN_DISABLE rather than
1918  *   a plain DISABLE restarter event.
1919  */
1920 void
1921 graph_enable_by_vertex(graph_vertex_t *vertex, int enable, int admin)
1922 {
1923         graph_vertex_t *v;
1924         int r;
1925 
1926         assert(MUTEX_HELD(&dgraph_lock));
1927         assert((vertex->gv_flags & GV_CONFIGURED));
1928 
1929         vertex->gv_flags = (vertex->gv_flags & ~GV_ENABLED) |
1930             (enable ? GV_ENABLED : 0);
1931 
1932         if (enable) {
1933                 if (vertex->gv_state != RESTARTER_STATE_OFFLINE &&
1934                     vertex->gv_state != RESTARTER_STATE_DEGRADED &&
1935                     vertex->gv_state != RESTARTER_STATE_ONLINE) {
1936                         /*
1937                          * In case the vertex was notified to go down,
1938                          * but now can return online, clear the _TOOFFLINE
1939                          * and _TODISABLE flags.
1940                          */
1941                         vertex->gv_flags &= ~GV_TOOFFLINE;
1942                         vertex->gv_flags &= ~GV_TODISABLE;
1943 
1944                         vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ENABLE);
1945                 }
1946 
1947                 /*
1948                  * Wait for state update from restarter before sending _START or
1949                  * _STOP.
1950                  */
1951 
1952                 return;
1953         }
1954 
1955         if (vertex->gv_state == RESTARTER_STATE_DISABLED)
1956                 return;
1957 
1958         if (!admin) {
1959                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_DISABLE);
1960 
1961                 /*
1962                  * Wait for state update from restarter before sending _START or
1963                  * _STOP.
1964                  */
1965 
1966                 return;
1967         }
1968 
1969         /*
1970          * If it is a DISABLE event requested by the administrator then we are
1971          * offlining the dependents first.
1972          */
1973 
1974         /*
1975          * Set GV_TOOFFLINE for the services we are offlining. We cannot
1976          * clear the GV_TOOFFLINE bits from all the services because
1977          * other DISABLE events might be handled at the same time.
1978          */
1979         vertex->gv_flags |= GV_TOOFFLINE;
1980 
1981         /* remember which vertex to disable... */
1982         vertex->gv_flags |= GV_TODISABLE;
1983 
1984         log_framework(LOG_DEBUG, "Marking in-subtree vertices before "
1985             "disabling %s.\n", vertex->gv_name);
1986 
1987         /* set GV_TOOFFLINE for its dependents */
1988         r = uu_list_walk(vertex->gv_dependents, (uu_walk_fn_t *)mark_subtree,
1989             NULL, 0);
1990         assert(r == 0);
1991 
1992         /* disable the instance now if there is nothing else to offline */
1993         if (insubtree_dependents_down(vertex) == B_TRUE) {
1994                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
1995                 return;
1996         }
1997 
1998         /*
1999          * This loop is similar to the one used for the graph reversal shutdown
2000          * and could be improved in term of performance for the subtree reversal
2001          * disable case.
2002          */
2003         for (v = uu_list_first(dgraph); v != NULL;
2004             v = uu_list_next(dgraph, v)) {
2005                 /* skip the vertex we are disabling for now */
2006                 if (v == vertex)
2007                         continue;
2008 
2009                 if (v->gv_type != GVT_INST ||
2010                     (v->gv_flags & GV_CONFIGURED) == 0 ||
2011                     (v->gv_flags & GV_ENABLED) == 0 ||
2012                     (v->gv_flags & GV_TOOFFLINE) == 0)
2013                         continue;
2014 
2015                 if ((v->gv_state != RESTARTER_STATE_ONLINE) &&
2016                     (v->gv_state != RESTARTER_STATE_DEGRADED)) {
2017                         /* continue if there is nothing to offline */
2018                         continue;
2019                 }
2020 
2021                 /*
2022                  * Instances which are up need to come down before we're
2023                  * done, but we can only offline the leaves here. An
2024                  * instance is a leaf when all its dependents are down.
2025                  */
2026                 if (insubtree_dependents_down(v) == B_TRUE) {
2027                         log_framework(LOG_DEBUG, "Offlining in-subtree "
2028                             "instance %s for %s.\n",
2029                             v->gv_name, vertex->gv_name);
2030                         offline_vertex(v);
2031                 }
2032         }
2033 }
2034 
2035 static int configure_vertex(graph_vertex_t *, scf_instance_t *);
2036 
2037 /*
2038  * Set the restarter for v to fmri_arg.  That is, make sure a vertex for
2039  * fmri_arg exists, make v depend on it, and send _ADD_INSTANCE for v.  If
2040  * v is already configured and fmri_arg indicates the current restarter, do
2041  * nothing.  If v is configured and fmri_arg is a new restarter, delete v's
2042  * dependency on the restarter, send _REMOVE_INSTANCE for v, and set the new
2043  * restarter.  Returns 0 on success, EINVAL if the FMRI is invalid,
2044  * ECONNABORTED if the repository connection is broken, and ELOOP
2045  * if the dependency would create a cycle.  In the last case, *pathp will
2046  * point to a -1-terminated array of ids which compose the path from v to
2047  * restarter_fmri.
2048  */
2049 int
2050 graph_change_restarter(graph_vertex_t *v, const char *fmri_arg, scf_handle_t *h,
2051     int **pathp)
2052 {
2053         char *restarter_fmri = NULL;
2054         graph_vertex_t *rv;
2055         int err;
2056         int id;
2057 
2058         assert(MUTEX_HELD(&dgraph_lock));
2059 
2060         if (fmri_arg[0] != '\0') {
2061                 err = fmri_canonify(fmri_arg, &restarter_fmri, B_TRUE);
2062                 if (err != 0) {
2063                         assert(err == EINVAL);
2064                         return (err);
2065                 }
2066         }
2067 
2068         if (restarter_fmri == NULL ||
2069             strcmp(restarter_fmri, SCF_SERVICE_STARTD) == 0) {
2070                 if (v->gv_flags & GV_CONFIGURED) {
2071                         if (v->gv_restarter_id == -1) {
2072                                 if (restarter_fmri != NULL)
2073                                         startd_free(restarter_fmri,
2074                                             max_scf_fmri_size);
2075                                 return (0);
2076                         }
2077 
2078                         graph_unset_restarter(v);
2079                 }
2080 
2081                 /* Master restarter, nothing to do. */
2082                 v->gv_restarter_id = -1;
2083                 v->gv_restarter_channel = NULL;
2084                 vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2085                 return (0);
2086         }
2087 
2088         if (v->gv_flags & GV_CONFIGURED) {
2089                 id = dict_lookup_byname(restarter_fmri);
2090                 if (id != -1 && v->gv_restarter_id == id) {
2091                         startd_free(restarter_fmri, max_scf_fmri_size);
2092                         return (0);
2093                 }
2094 
2095                 graph_unset_restarter(v);
2096         }
2097 
2098         err = graph_insert_vertex_unconfigured(restarter_fmri, GVT_INST, 0,
2099             RERR_NONE, &rv);
2100         startd_free(restarter_fmri, max_scf_fmri_size);
2101         assert(err == 0 || err == EEXIST);
2102 
2103         if (rv->gv_delegate_initialized == 0) {
2104                 if ((rv->gv_delegate_channel = restarter_protocol_init_delegate(
2105                     rv->gv_name)) == NULL)
2106                         return (EINVAL);
2107                 rv->gv_delegate_initialized = 1;
2108         }
2109         v->gv_restarter_id = rv->gv_id;
2110         v->gv_restarter_channel = rv->gv_delegate_channel;
2111 
2112         err = graph_insert_dependency(v, rv, pathp);
2113         if (err != 0) {
2114                 assert(err == ELOOP);
2115                 return (ELOOP);
2116         }
2117 
2118         vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2119 
2120         if (!(rv->gv_flags & GV_CONFIGURED)) {
2121                 scf_instance_t *inst;
2122 
2123                 err = libscf_fmri_get_instance(h, rv->gv_name, &inst);
2124                 switch (err) {
2125                 case 0:
2126                         err = configure_vertex(rv, inst);
2127                         scf_instance_destroy(inst);
2128                         switch (err) {
2129                         case 0:
2130                         case ECANCELED:
2131                                 break;
2132 
2133                         case ECONNABORTED:
2134                                 return (ECONNABORTED);
2135 
2136                         default:
2137                                 bad_error("configure_vertex", err);
2138                         }
2139                         break;
2140 
2141                 case ECONNABORTED:
2142                         return (ECONNABORTED);
2143 
2144                 case ENOENT:
2145                         break;
2146 
2147                 case ENOTSUP:
2148                         /*
2149                          * The fmri doesn't specify an instance - translate
2150                          * to EINVAL.
2151                          */
2152                         return (EINVAL);
2153 
2154                 case EINVAL:
2155                 default:
2156                         bad_error("libscf_fmri_get_instance", err);
2157                 }
2158         }
2159 
2160         return (0);
2161 }
2162 
2163 
2164 /*
2165  * Add all of the instances of the service named by fmri to the graph.
2166  * Returns
2167  *   0 - success
2168  *   ENOENT - service indicated by fmri does not exist
2169  *
2170  * In both cases *reboundp will be B_TRUE if the handle was rebound, or B_FALSE
2171  * otherwise.
2172  */
2173 static int
2174 add_service(const char *fmri, scf_handle_t *h, boolean_t *reboundp)
2175 {
2176         scf_service_t *svc;
2177         scf_instance_t *inst;
2178         scf_iter_t *iter;
2179         char *inst_fmri;
2180         int ret, r;
2181 
2182         *reboundp = B_FALSE;
2183 
2184         svc = safe_scf_service_create(h);
2185         inst = safe_scf_instance_create(h);
2186         iter = safe_scf_iter_create(h);
2187         inst_fmri = startd_alloc(max_scf_fmri_size);
2188 
2189 rebound:
2190         if (scf_handle_decode_fmri(h, fmri, NULL, svc, NULL, NULL, NULL,
2191             SCF_DECODE_FMRI_EXACT) != 0) {
2192                 switch (scf_error()) {
2193                 case SCF_ERROR_CONNECTION_BROKEN:
2194                 default:
2195                         libscf_handle_rebind(h);
2196                         *reboundp = B_TRUE;
2197                         goto rebound;
2198 
2199                 case SCF_ERROR_NOT_FOUND:
2200                         ret = ENOENT;
2201                         goto out;
2202 
2203                 case SCF_ERROR_INVALID_ARGUMENT:
2204                 case SCF_ERROR_CONSTRAINT_VIOLATED:
2205                 case SCF_ERROR_NOT_BOUND:
2206                 case SCF_ERROR_HANDLE_MISMATCH:
2207                         bad_error("scf_handle_decode_fmri", scf_error());
2208                 }
2209         }
2210 
2211         if (scf_iter_service_instances(iter, svc) != 0) {
2212                 switch (scf_error()) {
2213                 case SCF_ERROR_CONNECTION_BROKEN:
2214                 default:
2215                         libscf_handle_rebind(h);
2216                         *reboundp = B_TRUE;
2217                         goto rebound;
2218 
2219                 case SCF_ERROR_DELETED:
2220                         ret = ENOENT;
2221                         goto out;
2222 
2223                 case SCF_ERROR_HANDLE_MISMATCH:
2224                 case SCF_ERROR_NOT_BOUND:
2225                 case SCF_ERROR_NOT_SET:
2226                         bad_error("scf_iter_service_instances", scf_error());
2227                 }
2228         }
2229 
2230         for (;;) {
2231                 r = scf_iter_next_instance(iter, inst);
2232                 if (r == 0)
2233                         break;
2234                 if (r != 1) {
2235                         switch (scf_error()) {
2236                         case SCF_ERROR_CONNECTION_BROKEN:
2237                         default:
2238                                 libscf_handle_rebind(h);
2239                                 *reboundp = B_TRUE;
2240                                 goto rebound;
2241 
2242                         case SCF_ERROR_DELETED:
2243                                 ret = ENOENT;
2244                                 goto out;
2245 
2246                         case SCF_ERROR_HANDLE_MISMATCH:
2247                         case SCF_ERROR_NOT_BOUND:
2248                         case SCF_ERROR_NOT_SET:
2249                         case SCF_ERROR_INVALID_ARGUMENT:
2250                                 bad_error("scf_iter_next_instance",
2251                                     scf_error());
2252                         }
2253                 }
2254 
2255                 if (scf_instance_to_fmri(inst, inst_fmri, max_scf_fmri_size) <
2256                     0) {
2257                         switch (scf_error()) {
2258                         case SCF_ERROR_CONNECTION_BROKEN:
2259                                 libscf_handle_rebind(h);
2260                                 *reboundp = B_TRUE;
2261                                 goto rebound;
2262 
2263                         case SCF_ERROR_DELETED:
2264                                 continue;
2265 
2266                         case SCF_ERROR_NOT_BOUND:
2267                         case SCF_ERROR_NOT_SET:
2268                                 bad_error("scf_instance_to_fmri", scf_error());
2269                         }
2270                 }
2271 
2272                 r = dgraph_add_instance(inst_fmri, inst, B_FALSE);
2273                 switch (r) {
2274                 case 0:
2275                 case ECANCELED:
2276                         break;
2277 
2278                 case EEXIST:
2279                         continue;
2280 
2281                 case ECONNABORTED:
2282                         libscf_handle_rebind(h);
2283                         *reboundp = B_TRUE;
2284                         goto rebound;
2285 
2286                 case EINVAL:
2287                 default:
2288                         bad_error("dgraph_add_instance", r);
2289                 }
2290         }
2291 
2292         ret = 0;
2293 
2294 out:
2295         startd_free(inst_fmri, max_scf_fmri_size);
2296         scf_iter_destroy(iter);
2297         scf_instance_destroy(inst);
2298         scf_service_destroy(svc);
2299         return (ret);
2300 }
2301 
2302 struct depfmri_info {
2303         graph_vertex_t  *v;             /* GVT_GROUP vertex */
2304         gv_type_t       type;           /* type of dependency */
2305         const char      *inst_fmri;     /* FMRI of parental GVT_INST vert. */
2306         const char      *pg_name;       /* Name of dependency pg */
2307         scf_handle_t    *h;
2308         int             err;            /* return error code */
2309         int             **pathp;        /* return circular dependency path */
2310 };
2311 
2312 /*
2313  * Find or create a vertex for fmri and make info->v depend on it.
2314  * Returns
2315  *   0 - success
2316  *   nonzero - failure
2317  *
2318  * On failure, sets info->err to
2319  *   EINVAL - fmri is invalid
2320  *            fmri does not match info->type
2321  *   ELOOP - Adding the dependency creates a circular dependency.  *info->pathp
2322  *           will point to an array of the ids of the members of the cycle.
2323  *   ECONNABORTED - repository connection was broken
2324  *   ECONNRESET - succeeded, but repository connection was reset
2325  */
2326 static int
2327 process_dependency_fmri(const char *fmri, struct depfmri_info *info)
2328 {
2329         int err;
2330         graph_vertex_t *depgroup_v, *v;
2331         char *fmri_copy, *cfmri;
2332         size_t fmri_copy_sz;
2333         const char *scope, *service, *instance, *pg;
2334         scf_instance_t *inst;
2335         boolean_t rebound;
2336 
2337         assert(MUTEX_HELD(&dgraph_lock));
2338 
2339         /* Get or create vertex for FMRI */
2340         depgroup_v = info->v;
2341 
2342         if (strncmp(fmri, "file:", sizeof ("file:") - 1) == 0) {
2343                 if (info->type != GVT_FILE) {
2344                         log_framework(LOG_NOTICE,
2345                             "FMRI \"%s\" is not allowed for the \"%s\" "
2346                             "dependency's type of instance %s.\n", fmri,
2347                             info->pg_name, info->inst_fmri);
2348                         return (info->err = EINVAL);
2349                 }
2350 
2351                 err = graph_insert_vertex_unconfigured(fmri, info->type, 0,
2352                     RERR_NONE, &v);
2353                 switch (err) {
2354                 case 0:
2355                         break;
2356 
2357                 case EEXIST:
2358                         assert(v->gv_type == GVT_FILE);
2359                         break;
2360 
2361                 case EINVAL:            /* prevented above */
2362                 default:
2363                         bad_error("graph_insert_vertex_unconfigured", err);
2364                 }
2365         } else {
2366                 if (info->type != GVT_INST) {
2367                         log_framework(LOG_NOTICE,
2368                             "FMRI \"%s\" is not allowed for the \"%s\" "
2369                             "dependency's type of instance %s.\n", fmri,
2370                             info->pg_name, info->inst_fmri);
2371                         return (info->err = EINVAL);
2372                 }
2373 
2374                 /*
2375                  * We must canonify fmri & add a vertex for it.
2376                  */
2377                 fmri_copy_sz = strlen(fmri) + 1;
2378                 fmri_copy = startd_alloc(fmri_copy_sz);
2379                 (void) strcpy(fmri_copy, fmri);
2380 
2381                 /* Determine if the FMRI is a property group or instance */
2382                 if (scf_parse_svc_fmri(fmri_copy, &scope, &service,
2383                     &instance, &pg, NULL) != 0) {
2384                         startd_free(fmri_copy, fmri_copy_sz);
2385                         log_framework(LOG_NOTICE,
2386                             "Dependency \"%s\" of %s has invalid FMRI "
2387                             "\"%s\".\n", info->pg_name, info->inst_fmri,
2388                             fmri);
2389                         return (info->err = EINVAL);
2390                 }
2391 
2392                 if (service == NULL || pg != NULL) {
2393                         startd_free(fmri_copy, fmri_copy_sz);
2394                         log_framework(LOG_NOTICE,
2395                             "Dependency \"%s\" of %s does not designate a "
2396                             "service or instance.\n", info->pg_name,
2397                             info->inst_fmri);
2398                         return (info->err = EINVAL);
2399                 }
2400 
2401                 if (scope == NULL || strcmp(scope, SCF_SCOPE_LOCAL) == 0) {
2402                         cfmri = uu_msprintf("svc:/%s%s%s",
2403                             service, instance ? ":" : "", instance ? instance :
2404                             "");
2405                 } else {
2406                         cfmri = uu_msprintf("svc://%s/%s%s%s",
2407                             scope, service, instance ? ":" : "", instance ?
2408                             instance : "");
2409                 }
2410 
2411                 startd_free(fmri_copy, fmri_copy_sz);
2412 
2413                 err = graph_insert_vertex_unconfigured(cfmri, instance ?
2414                     GVT_INST : GVT_SVC, instance ? 0 : DEPGRP_REQUIRE_ANY,
2415                     RERR_NONE, &v);
2416                 uu_free(cfmri);
2417                 switch (err) {
2418                 case 0:
2419                         break;
2420 
2421                 case EEXIST:
2422                         /* Verify v. */
2423                         if (instance != NULL)
2424                                 assert(v->gv_type == GVT_INST);
2425                         else
2426                                 assert(v->gv_type == GVT_SVC);
2427                         break;
2428 
2429                 default:
2430                         bad_error("graph_insert_vertex_unconfigured", err);
2431                 }
2432         }
2433 
2434         /* Add dependency from depgroup_v to new vertex */
2435         info->err = graph_insert_dependency(depgroup_v, v, info->pathp);
2436         switch (info->err) {
2437         case 0:
2438                 break;
2439 
2440         case ELOOP:
2441                 return (ELOOP);
2442 
2443         default:
2444                 bad_error("graph_insert_dependency", info->err);
2445         }
2446 
2447         /* This must be after we insert the dependency, to avoid looping. */
2448         switch (v->gv_type) {
2449         case GVT_INST:
2450                 if ((v->gv_flags & GV_CONFIGURED) != 0)
2451                         break;
2452 
2453                 inst = safe_scf_instance_create(info->h);
2454 
2455                 rebound = B_FALSE;
2456 
2457 rebound:
2458                 err = libscf_lookup_instance(v->gv_name, inst);
2459                 switch (err) {
2460                 case 0:
2461                         err = configure_vertex(v, inst);
2462                         switch (err) {
2463                         case 0:
2464                         case ECANCELED:
2465                                 break;
2466 
2467                         case ECONNABORTED:
2468                                 libscf_handle_rebind(info->h);
2469                                 rebound = B_TRUE;
2470                                 goto rebound;
2471 
2472                         default:
2473                                 bad_error("configure_vertex", err);
2474                         }
2475                         break;
2476 
2477                 case ENOENT:
2478                         break;
2479 
2480                 case ECONNABORTED:
2481                         libscf_handle_rebind(info->h);
2482                         rebound = B_TRUE;
2483                         goto rebound;
2484 
2485                 case EINVAL:
2486                 case ENOTSUP:
2487                 default:
2488                         bad_error("libscf_fmri_get_instance", err);
2489                 }
2490 
2491                 scf_instance_destroy(inst);
2492 
2493                 if (rebound)
2494                         return (info->err = ECONNRESET);
2495                 break;
2496 
2497         case GVT_SVC:
2498                 (void) add_service(v->gv_name, info->h, &rebound);
2499                 if (rebound)
2500                         return (info->err = ECONNRESET);
2501         }
2502 
2503         return (0);
2504 }
2505 
2506 struct deppg_info {
2507         graph_vertex_t  *v;             /* GVT_INST vertex */
2508         int             err;            /* return error */
2509         int             **pathp;        /* return circular dependency path */
2510 };
2511 
2512 /*
2513  * Make info->v depend on a new GVT_GROUP node for this property group,
2514  * and then call process_dependency_fmri() for the values of the entity
2515  * property.  Return 0 on success, or if something goes wrong return nonzero
2516  * and set info->err to ECONNABORTED, EINVAL, or the error code returned by
2517  * process_dependency_fmri().
2518  */
2519 static int
2520 process_dependency_pg(scf_propertygroup_t *pg, struct deppg_info *info)
2521 {
2522         scf_handle_t *h;
2523         depgroup_type_t deptype;
2524         restarter_error_t rerr;
2525         struct depfmri_info linfo;
2526         char *fmri, *pg_name;
2527         size_t fmri_sz;
2528         graph_vertex_t *depgrp;
2529         scf_property_t *prop;
2530         int err;
2531         int empty;
2532         scf_error_t scferr;
2533         ssize_t len;
2534 
2535         assert(MUTEX_HELD(&dgraph_lock));
2536 
2537         h = scf_pg_handle(pg);
2538 
2539         pg_name = startd_alloc(max_scf_name_size);
2540 
2541         len = scf_pg_get_name(pg, pg_name, max_scf_name_size);
2542         if (len < 0) {
2543                 startd_free(pg_name, max_scf_name_size);
2544                 switch (scf_error()) {
2545                 case SCF_ERROR_CONNECTION_BROKEN:
2546                 default:
2547                         return (info->err = ECONNABORTED);
2548 
2549                 case SCF_ERROR_DELETED:
2550                         return (info->err = 0);
2551 
2552                 case SCF_ERROR_NOT_SET:
2553                         bad_error("scf_pg_get_name", scf_error());
2554                 }
2555         }
2556 
2557         /*
2558          * Skip over empty dependency groups.  Since dependency property
2559          * groups are updated atomically, they are either empty or
2560          * fully populated.
2561          */
2562         empty = depgroup_empty(h, pg);
2563         if (empty < 0) {
2564                 log_error(LOG_INFO,
2565                     "Error reading dependency group \"%s\" of %s: %s\n",
2566                     pg_name, info->v->gv_name, scf_strerror(scf_error()));
2567                 startd_free(pg_name, max_scf_name_size);
2568                 return (info->err = EINVAL);
2569 
2570         } else if (empty == 1) {
2571                 log_framework(LOG_DEBUG,
2572                     "Ignoring empty dependency group \"%s\" of %s\n",
2573                     pg_name, info->v->gv_name);
2574                 startd_free(pg_name, max_scf_name_size);
2575                 return (info->err = 0);
2576         }
2577 
2578         fmri_sz = strlen(info->v->gv_name) + 1 + len + 1;
2579         fmri = startd_alloc(fmri_sz);
2580 
2581         (void) snprintf(fmri, fmri_sz, "%s>%s", info->v->gv_name,
2582             pg_name);
2583 
2584         /* Validate the pg before modifying the graph */
2585         deptype = depgroup_read_grouping(h, pg);
2586         if (deptype == DEPGRP_UNSUPPORTED) {
2587                 log_error(LOG_INFO,
2588                     "Dependency \"%s\" of %s has an unknown grouping value.\n",
2589                     pg_name, info->v->gv_name);
2590                 startd_free(fmri, fmri_sz);
2591                 startd_free(pg_name, max_scf_name_size);
2592                 return (info->err = EINVAL);
2593         }
2594 
2595         rerr = depgroup_read_restart(h, pg);
2596         if (rerr == RERR_UNSUPPORTED) {
2597                 log_error(LOG_INFO,
2598                     "Dependency \"%s\" of %s has an unknown restart_on value."
2599                     "\n", pg_name, info->v->gv_name);
2600                 startd_free(fmri, fmri_sz);
2601                 startd_free(pg_name, max_scf_name_size);
2602                 return (info->err = EINVAL);
2603         }
2604 
2605         prop = safe_scf_property_create(h);
2606 
2607         if (scf_pg_get_property(pg, SCF_PROPERTY_ENTITIES, prop) != 0) {
2608                 scferr = scf_error();
2609                 scf_property_destroy(prop);
2610                 if (scferr == SCF_ERROR_DELETED) {
2611                         startd_free(fmri, fmri_sz);
2612                         startd_free(pg_name, max_scf_name_size);
2613                         return (info->err = 0);
2614                 } else if (scferr != SCF_ERROR_NOT_FOUND) {
2615                         startd_free(fmri, fmri_sz);
2616                         startd_free(pg_name, max_scf_name_size);
2617                         return (info->err = ECONNABORTED);
2618                 }
2619 
2620                 log_error(LOG_INFO,
2621                     "Dependency \"%s\" of %s is missing a \"%s\" property.\n",
2622                     pg_name, info->v->gv_name, SCF_PROPERTY_ENTITIES);
2623 
2624                 startd_free(fmri, fmri_sz);
2625                 startd_free(pg_name, max_scf_name_size);
2626 
2627                 return (info->err = EINVAL);
2628         }
2629 
2630         /* Create depgroup vertex for pg */
2631         err = graph_insert_vertex_unconfigured(fmri, GVT_GROUP, deptype,
2632             rerr, &depgrp);
2633         assert(err == 0);
2634         startd_free(fmri, fmri_sz);
2635 
2636         /* Add dependency from inst vertex to new vertex */
2637         err = graph_insert_dependency(info->v, depgrp, info->pathp);
2638         /* ELOOP can't happen because this should be a new vertex */
2639         assert(err == 0);
2640 
2641         linfo.v = depgrp;
2642         linfo.type = depgroup_read_scheme(h, pg);
2643         linfo.inst_fmri = info->v->gv_name;
2644         linfo.pg_name = pg_name;
2645         linfo.h = h;
2646         linfo.err = 0;
2647         linfo.pathp = info->pathp;
2648         err = walk_property_astrings(prop, (callback_t)process_dependency_fmri,
2649             &linfo);
2650 
2651         scf_property_destroy(prop);
2652         startd_free(pg_name, max_scf_name_size);
2653 
2654         switch (err) {
2655         case 0:
2656         case EINTR:
2657                 return (info->err = linfo.err);
2658 
2659         case ECONNABORTED:
2660         case EINVAL:
2661                 return (info->err = err);
2662 
2663         case ECANCELED:
2664                 return (info->err = 0);
2665 
2666         case ECONNRESET:
2667                 return (info->err = ECONNABORTED);
2668 
2669         default:
2670                 bad_error("walk_property_astrings", err);
2671                 /* NOTREACHED */
2672         }
2673 }
2674 
2675 /*
2676  * Build the dependency info for v from the repository.  Returns 0 on success,
2677  * ECONNABORTED on repository disconnection, EINVAL if the repository
2678  * configuration is invalid, and ELOOP if a dependency would cause a cycle.
2679  * In the last case, *pathp will point to a -1-terminated array of ids which
2680  * constitute the rest of the dependency cycle.
2681  */
2682 static int
2683 set_dependencies(graph_vertex_t *v, scf_instance_t *inst, int **pathp)
2684 {
2685         struct deppg_info info;
2686         int err;
2687         uint_t old_configured;
2688 
2689         assert(MUTEX_HELD(&dgraph_lock));
2690 
2691         /*
2692          * Mark the vertex as configured during dependency insertion to avoid
2693          * dependency cycles (which can appear in the graph if one of the
2694          * vertices is an exclusion-group).
2695          */
2696         old_configured = v->gv_flags & GV_CONFIGURED;
2697         v->gv_flags |= GV_CONFIGURED;
2698 
2699         info.err = 0;
2700         info.v = v;
2701         info.pathp = pathp;
2702 
2703         err = walk_dependency_pgs(inst, (callback_t)process_dependency_pg,
2704             &info);
2705 
2706         if (!old_configured)
2707                 v->gv_flags &= ~GV_CONFIGURED;
2708 
2709         switch (err) {
2710         case 0:
2711         case EINTR:
2712                 return (info.err);
2713 
2714         case ECONNABORTED:
2715                 return (ECONNABORTED);
2716 
2717         case ECANCELED:
2718                 /* Should get delete event, so return 0. */
2719                 return (0);
2720 
2721         default:
2722                 bad_error("walk_dependency_pgs", err);
2723                 /* NOTREACHED */
2724         }
2725 }
2726 
2727 
2728 static void
2729 handle_cycle(const char *fmri, int *path)
2730 {
2731         const char *cp;
2732         size_t sz;
2733 
2734         assert(MUTEX_HELD(&dgraph_lock));
2735 
2736         path_to_str(path, (char **)&cp, &sz);
2737 
2738         log_error(LOG_ERR, "Transitioning %s to maintenance "
2739             "because it completes a dependency cycle (see svcs -xv for "
2740             "details):\n%s", fmri ? fmri : "?", cp);
2741 
2742         startd_free((void *)cp, sz);
2743 }
2744 
2745 /*
2746  * Increment the vertex's reference count to prevent the vertex removal
2747  * from the dgraph.
2748  */
2749 static void
2750 vertex_ref(graph_vertex_t *v)
2751 {
2752         assert(MUTEX_HELD(&dgraph_lock));
2753 
2754         v->gv_refs++;
2755 }
2756 
2757 /*
2758  * Decrement the vertex's reference count and remove the vertex from
2759  * the dgraph when possible.
2760  *
2761  * Return VERTEX_REMOVED when the vertex has been removed otherwise
2762  * return VERTEX_INUSE.
2763  */
2764 static int
2765 vertex_unref(graph_vertex_t *v)
2766 {
2767         assert(MUTEX_HELD(&dgraph_lock));
2768         assert(v->gv_refs > 0);
2769 
2770         v->gv_refs--;
2771 
2772         return (free_if_unrefed(v));
2773 }
2774 
2775 /*
2776  * When run on the dependencies of a vertex, populates list with
2777  * graph_edge_t's which point to the service vertices or the instance
2778  * vertices (no GVT_GROUP nodes) on which the vertex depends.
2779  *
2780  * Increment the vertex's reference count once the vertex is inserted
2781  * in the list. The vertex won't be able to be deleted from the dgraph
2782  * while it is referenced.
2783  */
2784 static int
2785 append_svcs_or_insts(graph_edge_t *e, uu_list_t *list)
2786 {
2787         graph_vertex_t *v = e->ge_vertex;
2788         graph_edge_t *new;
2789         int r;
2790 
2791         switch (v->gv_type) {
2792         case GVT_INST:
2793         case GVT_SVC:
2794                 break;
2795 
2796         case GVT_GROUP:
2797                 r = uu_list_walk(v->gv_dependencies,
2798                     (uu_walk_fn_t *)append_svcs_or_insts, list, 0);
2799                 assert(r == 0);
2800                 return (UU_WALK_NEXT);
2801 
2802         case GVT_FILE:
2803                 return (UU_WALK_NEXT);
2804 
2805         default:
2806 #ifndef NDEBUG
2807                 uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
2808                     __LINE__, v->gv_type);
2809 #endif
2810                 abort();
2811         }
2812 
2813         new = startd_alloc(sizeof (*new));
2814         new->ge_vertex = v;
2815         uu_list_node_init(new, &new->ge_link, graph_edge_pool);
2816         r = uu_list_insert_before(list, NULL, new);
2817         assert(r == 0);
2818 
2819         /*
2820          * Because we are inserting the vertex in a list, we don't want
2821          * the vertex to be freed while the list is in use. In order to
2822          * achieve that, increment the vertex's reference count.
2823          */
2824         vertex_ref(v);
2825 
2826         return (UU_WALK_NEXT);
2827 }
2828 
2829 static boolean_t
2830 should_be_in_subgraph(graph_vertex_t *v)
2831 {
2832         graph_edge_t *e;
2833 
2834         if (v == milestone)
2835                 return (B_TRUE);
2836 
2837         /*
2838          * v is in the subgraph if any of its dependents are in the subgraph.
2839          * Except for EXCLUDE_ALL dependents.  And OPTIONAL dependents only
2840          * count if we're enabled.
2841          */
2842         for (e = uu_list_first(v->gv_dependents);
2843             e != NULL;
2844             e = uu_list_next(v->gv_dependents, e)) {
2845                 graph_vertex_t *dv = e->ge_vertex;
2846 
2847                 if (!(dv->gv_flags & GV_INSUBGRAPH))
2848                         continue;
2849 
2850                 /*
2851                  * Don't include instances that are optional and disabled.
2852                  */
2853                 if (v->gv_type == GVT_INST && dv->gv_type == GVT_SVC) {
2854 
2855                         int in = 0;
2856                         graph_edge_t *ee;
2857 
2858                         for (ee = uu_list_first(dv->gv_dependents);
2859                             ee != NULL;
2860                             ee = uu_list_next(dv->gv_dependents, ee)) {
2861 
2862                                 graph_vertex_t *ddv = e->ge_vertex;
2863 
2864                                 if (ddv->gv_type == GVT_GROUP &&
2865                                     ddv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2866                                         continue;
2867 
2868                                 if (ddv->gv_type == GVT_GROUP &&
2869                                     ddv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2870                                     !(v->gv_flags & GV_ENBLD_NOOVR))
2871                                         continue;
2872 
2873                                 in = 1;
2874                         }
2875                         if (!in)
2876                                 continue;
2877                 }
2878                 if (v->gv_type == GVT_INST &&
2879                     dv->gv_type == GVT_GROUP &&
2880                     dv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2881                     !(v->gv_flags & GV_ENBLD_NOOVR))
2882                         continue;
2883 
2884                 /* Don't include excluded services and instances */
2885                 if (dv->gv_type == GVT_GROUP &&
2886                     dv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2887                         continue;
2888 
2889                 return (B_TRUE);
2890         }
2891 
2892         return (B_FALSE);
2893 }
2894 
2895 /*
2896  * Ensures that GV_INSUBGRAPH is set properly for v and its descendents.  If
2897  * any bits change, manipulate the repository appropriately.  Returns 0 or
2898  * ECONNABORTED.
2899  */
2900 static int
2901 eval_subgraph(graph_vertex_t *v, scf_handle_t *h)
2902 {
2903         boolean_t old = (v->gv_flags & GV_INSUBGRAPH) != 0;
2904         boolean_t new;
2905         graph_edge_t *e;
2906         scf_instance_t *inst;
2907         int ret = 0, r;
2908 
2909         assert(milestone != NULL && milestone != MILESTONE_NONE);
2910 
2911         new = should_be_in_subgraph(v);
2912 
2913         if (new == old)
2914                 return (0);
2915 
2916         log_framework(LOG_DEBUG, new ? "Adding %s to the subgraph.\n" :
2917             "Removing %s from the subgraph.\n", v->gv_name);
2918 
2919         v->gv_flags = (v->gv_flags & ~GV_INSUBGRAPH) |
2920             (new ? GV_INSUBGRAPH : 0);
2921 
2922         if (v->gv_type == GVT_INST && (v->gv_flags & GV_CONFIGURED)) {
2923                 int err;
2924 
2925 get_inst:
2926                 err = libscf_fmri_get_instance(h, v->gv_name, &inst);
2927                 if (err != 0) {
2928                         switch (err) {
2929                         case ECONNABORTED:
2930                                 libscf_handle_rebind(h);
2931                                 ret = ECONNABORTED;
2932                                 goto get_inst;
2933 
2934                         case ENOENT:
2935                                 break;
2936 
2937                         case EINVAL:
2938                         case ENOTSUP:
2939                         default:
2940                                 bad_error("libscf_fmri_get_instance", err);
2941                         }
2942                 } else {
2943                         const char *f;
2944 
2945                         if (new) {
2946                                 err = libscf_delete_enable_ovr(inst);
2947                                 f = "libscf_delete_enable_ovr";
2948                         } else {
2949                                 err = libscf_set_enable_ovr(inst, 0);
2950                                 f = "libscf_set_enable_ovr";
2951                         }
2952                         scf_instance_destroy(inst);
2953                         switch (err) {
2954                         case 0:
2955                         case ECANCELED:
2956                                 break;
2957 
2958                         case ECONNABORTED:
2959                                 libscf_handle_rebind(h);
2960                                 /*
2961                                  * We must continue so the graph is updated,
2962                                  * but we must return ECONNABORTED so any
2963                                  * libscf state held by any callers is reset.
2964                                  */
2965                                 ret = ECONNABORTED;
2966                                 goto get_inst;
2967 
2968                         case EROFS:
2969                         case EPERM:
2970                                 log_error(LOG_WARNING,
2971                                     "Could not set %s/%s for %s: %s.\n",
2972                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
2973                                     v->gv_name, strerror(err));
2974                                 break;
2975 
2976                         default:
2977                                 bad_error(f, err);
2978                         }
2979                 }
2980         }
2981 
2982         for (e = uu_list_first(v->gv_dependencies);
2983             e != NULL;
2984             e = uu_list_next(v->gv_dependencies, e)) {
2985                 r = eval_subgraph(e->ge_vertex, h);
2986                 if (r != 0) {
2987                         assert(r == ECONNABORTED);
2988                         ret = ECONNABORTED;
2989                 }
2990         }
2991 
2992         return (ret);
2993 }
2994 
2995 /*
2996  * Delete the (property group) dependencies of v & create new ones based on
2997  * inst.  If doing so would create a cycle, log a message and put the instance
2998  * into maintenance.  Update GV_INSUBGRAPH flags as necessary.  Returns 0 or
2999  * ECONNABORTED.
3000  */
3001 int
3002 refresh_vertex(graph_vertex_t *v, scf_instance_t *inst)
3003 {
3004         int err;
3005         int *path;
3006         char *fmri;
3007         int r;
3008         scf_handle_t *h = scf_instance_handle(inst);
3009         uu_list_t *old_deps;
3010         int ret = 0;
3011         graph_edge_t *e;
3012         graph_vertex_t *vv;
3013 
3014         assert(MUTEX_HELD(&dgraph_lock));
3015         assert(v->gv_type == GVT_INST);
3016 
3017         log_framework(LOG_DEBUG, "Graph engine: Refreshing %s.\n", v->gv_name);
3018 
3019         if (milestone > MILESTONE_NONE) {
3020                 /*
3021                  * In case some of v's dependencies are being deleted we must
3022                  * make a list of them now for GV_INSUBGRAPH-flag evaluation
3023                  * after the new dependencies are in place.
3024                  */
3025                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
3026 
3027                 err = uu_list_walk(v->gv_dependencies,
3028                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
3029                 assert(err == 0);
3030         }
3031 
3032         delete_instance_dependencies(v, B_FALSE);
3033 
3034         err = set_dependencies(v, inst, &path);
3035         switch (err) {
3036         case 0:
3037                 break;
3038 
3039         case ECONNABORTED:
3040                 ret = err;
3041                 goto out;
3042 
3043         case EINVAL:
3044         case ELOOP:
3045                 r = libscf_instance_get_fmri(inst, &fmri);
3046                 switch (r) {
3047                 case 0:
3048                         break;
3049 
3050                 case ECONNABORTED:
3051                         ret = ECONNABORTED;
3052                         goto out;
3053 
3054                 case ECANCELED:
3055                         ret = 0;
3056                         goto out;
3057 
3058                 default:
3059                         bad_error("libscf_instance_get_fmri", r);
3060                 }
3061 
3062                 if (err == EINVAL) {
3063                         log_error(LOG_ERR, "Transitioning %s "
3064                             "to maintenance due to misconfiguration.\n",
3065                             fmri ? fmri : "?");
3066                         vertex_send_event(v,
3067                             RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY);
3068                 } else {
3069                         handle_cycle(fmri, path);
3070                         vertex_send_event(v,
3071                             RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE);
3072                 }
3073                 startd_free(fmri, max_scf_fmri_size);
3074                 ret = 0;
3075                 goto out;
3076 
3077         default:
3078                 bad_error("set_dependencies", err);
3079         }
3080 
3081         if (milestone > MILESTONE_NONE) {
3082                 boolean_t aborted = B_FALSE;
3083 
3084                 for (e = uu_list_first(old_deps);
3085                     e != NULL;
3086                     e = uu_list_next(old_deps, e)) {
3087                         vv = e->ge_vertex;
3088 
3089                         if (vertex_unref(vv) == VERTEX_INUSE &&
3090                             eval_subgraph(vv, h) == ECONNABORTED)
3091                                 aborted = B_TRUE;
3092                 }
3093 
3094                 for (e = uu_list_first(v->gv_dependencies);
3095                     e != NULL;
3096                     e = uu_list_next(v->gv_dependencies, e)) {
3097                         if (eval_subgraph(e->ge_vertex, h) ==
3098                             ECONNABORTED)
3099                                 aborted = B_TRUE;
3100                 }
3101 
3102                 if (aborted) {
3103                         ret = ECONNABORTED;
3104                         goto out;
3105                 }
3106         }
3107 
3108         graph_start_if_satisfied(v);
3109 
3110         ret = 0;
3111 
3112 out:
3113         if (milestone > MILESTONE_NONE) {
3114                 void *cookie = NULL;
3115 
3116                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL)
3117                         startd_free(e, sizeof (*e));
3118 
3119                 uu_list_destroy(old_deps);
3120         }
3121 
3122         return (ret);
3123 }
3124 
3125 /*
3126  * Set up v according to inst.  That is, make sure it depends on its
3127  * restarter and set up its dependencies.  Send the ADD_INSTANCE command to
3128  * the restarter, and send ENABLE or DISABLE as appropriate.
3129  *
3130  * Returns 0 on success, ECONNABORTED on repository disconnection, or
3131  * ECANCELED if inst is deleted.
3132  */
3133 static int
3134 configure_vertex(graph_vertex_t *v, scf_instance_t *inst)
3135 {
3136         scf_handle_t *h;
3137         scf_propertygroup_t *pg;
3138         scf_snapshot_t *snap;
3139         char *restarter_fmri = startd_alloc(max_scf_value_size);
3140         int enabled, enabled_ovr;
3141         int err;
3142         int *path;
3143         int deathrow;
3144         int32_t tset;
3145 
3146         restarter_fmri[0] = '\0';
3147 
3148         assert(MUTEX_HELD(&dgraph_lock));
3149         assert(v->gv_type == GVT_INST);
3150         assert((v->gv_flags & GV_CONFIGURED) == 0);
3151 
3152         /* GV_INSUBGRAPH should already be set properly. */
3153         assert(should_be_in_subgraph(v) ==
3154             ((v->gv_flags & GV_INSUBGRAPH) != 0));
3155 
3156         /*
3157          * If the instance fmri is in the deathrow list then set the
3158          * GV_DEATHROW flag on the vertex and create and set to true the
3159          * SCF_PROPERTY_DEATHROW boolean property in the non-persistent
3160          * repository for this instance fmri.
3161          */
3162         if ((v->gv_flags & GV_DEATHROW) ||
3163             (is_fmri_in_deathrow(v->gv_name) == B_TRUE)) {
3164                 if ((v->gv_flags & GV_DEATHROW) == 0) {
3165                         /*
3166                          * Set flag GV_DEATHROW, create and set to true
3167                          * the SCF_PROPERTY_DEATHROW property in the
3168                          * non-persistent repository for this instance fmri.
3169                          */
3170                         v->gv_flags |= GV_DEATHROW;
3171 
3172                         switch (err = libscf_set_deathrow(inst, 1)) {
3173                         case 0:
3174                                 break;
3175 
3176                         case ECONNABORTED:
3177                         case ECANCELED:
3178                                 startd_free(restarter_fmri, max_scf_value_size);
3179                                 return (err);
3180 
3181                         case EROFS:
3182                                 log_error(LOG_WARNING, "Could not set %s/%s "
3183                                     "for deathrow %s: %s.\n",
3184                                     SCF_PG_DEATHROW, SCF_PROPERTY_DEATHROW,
3185                                     v->gv_name, strerror(err));
3186                                 break;
3187 
3188                         case EPERM:
3189                                 uu_die("Permission denied.\n");
3190                                 /* NOTREACHED */
3191 
3192                         default:
3193                                 bad_error("libscf_set_deathrow", err);
3194                         }
3195                         log_framework(LOG_DEBUG, "Deathrow, graph set %s.\n",
3196                             v->gv_name);
3197                 }
3198                 startd_free(restarter_fmri, max_scf_value_size);
3199                 return (0);
3200         }
3201 
3202         h = scf_instance_handle(inst);
3203 
3204         /*
3205          * Using a temporary deathrow boolean property, set through
3206          * libscf_set_deathrow(), only for fmris on deathrow, is necessary
3207          * because deathrow_fini() may already have been called, and in case
3208          * of a refresh, GV_DEATHROW may need to be set again.
3209          * libscf_get_deathrow() sets deathrow to 1 only if this instance
3210          * has a temporary boolean property named 'deathrow' valued true
3211          * in a property group 'deathrow', -1 or 0 in all other cases.
3212          */
3213         err = libscf_get_deathrow(h, inst, &deathrow);
3214         switch (err) {
3215         case 0:
3216                 break;
3217 
3218         case ECONNABORTED:
3219         case ECANCELED:
3220                 startd_free(restarter_fmri, max_scf_value_size);
3221                 return (err);
3222 
3223         default:
3224                 bad_error("libscf_get_deathrow", err);
3225         }
3226 
3227         if (deathrow == 1) {
3228                 v->gv_flags |= GV_DEATHROW;
3229                 startd_free(restarter_fmri, max_scf_value_size);
3230                 return (0);
3231         }
3232 
3233         log_framework(LOG_DEBUG, "Graph adding %s.\n", v->gv_name);
3234 
3235         /*
3236          * If the instance does not have a restarter property group,
3237          * initialize its state to uninitialized/none, in case the restarter
3238          * is not enabled.
3239          */
3240         pg = safe_scf_pg_create(h);
3241 
3242         if (scf_instance_get_pg(inst, SCF_PG_RESTARTER, pg) != 0) {
3243                 instance_data_t idata;
3244                 uint_t count = 0, msecs = ALLOC_DELAY;
3245 
3246                 switch (scf_error()) {
3247                 case SCF_ERROR_NOT_FOUND:
3248                         break;
3249 
3250                 case SCF_ERROR_CONNECTION_BROKEN:
3251                 default:
3252                         scf_pg_destroy(pg);
3253                         startd_free(restarter_fmri, max_scf_value_size);
3254                         return (ECONNABORTED);
3255 
3256                 case SCF_ERROR_DELETED:
3257                         scf_pg_destroy(pg);
3258                         startd_free(restarter_fmri, max_scf_value_size);
3259                         return (ECANCELED);
3260 
3261                 case SCF_ERROR_NOT_SET:
3262                         bad_error("scf_instance_get_pg", scf_error());
3263                 }
3264 
3265                 switch (err = libscf_instance_get_fmri(inst,
3266                     (char **)&idata.i_fmri)) {
3267                 case 0:
3268                         break;
3269 
3270                 case ECONNABORTED:
3271                 case ECANCELED:
3272                         scf_pg_destroy(pg);
3273                         startd_free(restarter_fmri, max_scf_value_size);
3274                         return (err);
3275 
3276                 default:
3277                         bad_error("libscf_instance_get_fmri", err);
3278                 }
3279 
3280                 idata.i_state = RESTARTER_STATE_NONE;
3281                 idata.i_next_state = RESTARTER_STATE_NONE;
3282 
3283 init_state:
3284                 switch (err = _restarter_commit_states(h, &idata,
3285                     RESTARTER_STATE_UNINIT, RESTARTER_STATE_NONE,
3286                     restarter_get_str_short(restarter_str_insert_in_graph))) {
3287                 case 0:
3288                         break;
3289 
3290                 case ENOMEM:
3291                         ++count;
3292                         if (count < ALLOC_RETRY) {
3293                                 (void) poll(NULL, 0, msecs);
3294                                 msecs *= ALLOC_DELAY_MULT;
3295                                 goto init_state;
3296                         }
3297 
3298                         uu_die("Insufficient memory.\n");
3299                         /* NOTREACHED */
3300 
3301                 case ECONNABORTED:
3302                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3303                         scf_pg_destroy(pg);
3304                         startd_free(restarter_fmri, max_scf_value_size);
3305                         return (ECONNABORTED);
3306 
3307                 case ENOENT:
3308                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3309                         scf_pg_destroy(pg);
3310                         startd_free(restarter_fmri, max_scf_value_size);
3311                         return (ECANCELED);
3312 
3313                 case EPERM:
3314                 case EACCES:
3315                 case EROFS:
3316                         log_error(LOG_NOTICE, "Could not initialize state for "
3317                             "%s: %s.\n", idata.i_fmri, strerror(err));
3318                         break;
3319 
3320                 case EINVAL:
3321                 default:
3322                         bad_error("_restarter_commit_states", err);
3323                 }
3324 
3325                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3326         }
3327 
3328         scf_pg_destroy(pg);
3329 
3330         if (milestone != NULL) {
3331                 /*
3332                  * Make sure the enable-override is set properly before we
3333                  * read whether we should be enabled.
3334                  */
3335                 if (milestone == MILESTONE_NONE ||
3336                     !(v->gv_flags & GV_INSUBGRAPH)) {
3337                         /*
3338                          * This might seem unjustified after the milestone
3339                          * transition has completed (non_subgraph_svcs == 0),
3340                          * but it's important because when we boot to
3341                          * a milestone, we set the milestone before populating
3342                          * the graph, and all of the new non-subgraph services
3343                          * need to be disabled here.
3344                          */
3345                         switch (err = libscf_set_enable_ovr(inst, 0)) {
3346                         case 0:
3347                                 break;
3348 
3349                         case ECONNABORTED:
3350                         case ECANCELED:
3351                                 startd_free(restarter_fmri, max_scf_value_size);
3352                                 return (err);
3353 
3354                         case EROFS:
3355                                 log_error(LOG_WARNING,
3356                                     "Could not set %s/%s for %s: %s.\n",
3357                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
3358                                     v->gv_name, strerror(err));
3359                                 break;
3360 
3361                         case EPERM:
3362                                 uu_die("Permission denied.\n");
3363                                 /* NOTREACHED */
3364 
3365                         default:
3366                                 bad_error("libscf_set_enable_ovr", err);
3367                         }
3368                 } else {
3369                         assert(v->gv_flags & GV_INSUBGRAPH);
3370                         switch (err = libscf_delete_enable_ovr(inst)) {
3371                         case 0:
3372                                 break;
3373 
3374                         case ECONNABORTED:
3375                         case ECANCELED:
3376                                 startd_free(restarter_fmri, max_scf_value_size);
3377                                 return (err);
3378 
3379                         case EPERM:
3380                                 uu_die("Permission denied.\n");
3381                                 /* NOTREACHED */
3382 
3383                         default:
3384                                 bad_error("libscf_delete_enable_ovr", err);
3385                         }
3386                 }
3387         }
3388 
3389         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
3390             &enabled_ovr, &restarter_fmri);
3391         switch (err) {
3392         case 0:
3393                 break;
3394 
3395         case ECONNABORTED:
3396         case ECANCELED:
3397                 startd_free(restarter_fmri, max_scf_value_size);
3398                 return (err);
3399 
3400         case ENOENT:
3401                 log_framework(LOG_DEBUG,
3402                     "Ignoring %s because it has no general property group.\n",
3403                     v->gv_name);
3404                 startd_free(restarter_fmri, max_scf_value_size);
3405                 return (0);
3406 
3407         default:
3408                 bad_error("libscf_get_basic_instance_data", err);
3409         }
3410 
3411         if ((tset = libscf_get_stn_tset(inst)) == -1) {
3412                 log_framework(LOG_WARNING,
3413                     "Failed to get notification parameters for %s: %s\n",
3414                     v->gv_name, scf_strerror(scf_error()));
3415                 v->gv_stn_tset = 0;
3416         } else {
3417                 v->gv_stn_tset = tset;
3418         }
3419         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
3420                 stn_global = v->gv_stn_tset;
3421 
3422         if (enabled == -1) {
3423                 startd_free(restarter_fmri, max_scf_value_size);
3424                 return (0);
3425         }
3426 
3427         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
3428             (enabled ? GV_ENBLD_NOOVR : 0);
3429 
3430         if (enabled_ovr != -1)
3431                 enabled = enabled_ovr;
3432 
3433         v->gv_state = RESTARTER_STATE_UNINIT;
3434 
3435         snap = libscf_get_or_make_running_snapshot(inst, v->gv_name, B_TRUE);
3436         scf_snapshot_destroy(snap);
3437 
3438         /* Set up the restarter. (Sends _ADD_INSTANCE on success.) */
3439         err = graph_change_restarter(v, restarter_fmri, h, &path);
3440         if (err != 0) {
3441                 instance_data_t idata;
3442                 uint_t count = 0, msecs = ALLOC_DELAY;
3443                 restarter_str_t reason;
3444 
3445                 if (err == ECONNABORTED) {
3446                         startd_free(restarter_fmri, max_scf_value_size);
3447                         return (err);
3448                 }
3449 
3450                 assert(err == EINVAL || err == ELOOP);
3451 
3452                 if (err == EINVAL) {
3453                         log_framework(LOG_ERR, emsg_invalid_restarter,
3454                             v->gv_name, restarter_fmri);
3455                         reason = restarter_str_invalid_restarter;
3456                 } else {
3457                         handle_cycle(v->gv_name, path);
3458                         reason = restarter_str_dependency_cycle;
3459                 }
3460 
3461                 startd_free(restarter_fmri, max_scf_value_size);
3462 
3463                 /*
3464                  * We didn't register the instance with the restarter, so we
3465                  * must set maintenance mode ourselves.
3466                  */
3467                 err = libscf_instance_get_fmri(inst, (char **)&idata.i_fmri);
3468                 if (err != 0) {
3469                         assert(err == ECONNABORTED || err == ECANCELED);
3470                         return (err);
3471                 }
3472 
3473                 idata.i_state = RESTARTER_STATE_NONE;
3474                 idata.i_next_state = RESTARTER_STATE_NONE;
3475 
3476 set_maint:
3477                 switch (err = _restarter_commit_states(h, &idata,
3478                     RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE,
3479                     restarter_get_str_short(reason))) {
3480                 case 0:
3481                         break;
3482 
3483                 case ENOMEM:
3484                         ++count;
3485                         if (count < ALLOC_RETRY) {
3486                                 (void) poll(NULL, 0, msecs);
3487                                 msecs *= ALLOC_DELAY_MULT;
3488                                 goto set_maint;
3489                         }
3490 
3491                         uu_die("Insufficient memory.\n");
3492                         /* NOTREACHED */
3493 
3494                 case ECONNABORTED:
3495                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3496                         return (ECONNABORTED);
3497 
3498                 case ENOENT:
3499                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3500                         return (ECANCELED);
3501 
3502                 case EPERM:
3503                 case EACCES:
3504                 case EROFS:
3505                         log_error(LOG_NOTICE, "Could not initialize state for "
3506                             "%s: %s.\n", idata.i_fmri, strerror(err));
3507                         break;
3508 
3509                 case EINVAL:
3510                 default:
3511                         bad_error("_restarter_commit_states", err);
3512                 }
3513 
3514                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3515 
3516                 v->gv_state = RESTARTER_STATE_MAINT;
3517 
3518                 goto out;
3519         }
3520         startd_free(restarter_fmri, max_scf_value_size);
3521 
3522         /* Add all the other dependencies. */
3523         err = refresh_vertex(v, inst);
3524         if (err != 0) {
3525                 assert(err == ECONNABORTED);
3526                 return (err);
3527         }
3528 
3529 out:
3530         v->gv_flags |= GV_CONFIGURED;
3531 
3532         graph_enable_by_vertex(v, enabled, 0);
3533 
3534         return (0);
3535 }
3536 
3537 
3538 static void
3539 kill_user_procs(void)
3540 {
3541         (void) fputs("svc.startd: Killing user processes.\n", stdout);
3542 
3543         /*
3544          * Despite its name, killall's role is to get select user processes--
3545          * basically those representing terminal-based logins-- to die.  Victims
3546          * are located by killall in the utmp database.  Since these are most
3547          * often shell based logins, and many shells mask SIGTERM (but are
3548          * responsive to SIGHUP) we first HUP and then shortly thereafter
3549          * kill -9.
3550          */
3551         (void) fork_with_timeout("/usr/sbin/killall HUP", 1, 5);
3552         (void) fork_with_timeout("/usr/sbin/killall KILL", 1, 5);
3553 
3554         /*
3555          * Note the selection of user id's 0, 1 and 15, subsequently
3556          * inverted by -v.  15 is reserved for dladmd.  Yes, this is a
3557          * kludge-- a better policy is needed.
3558          *
3559          * Note that fork_with_timeout will only wait out the 1 second
3560          * "grace time" if pkill actually returns 0.  So if there are
3561          * no matches, this will run to completion much more quickly.
3562          */
3563         (void) fork_with_timeout("/usr/bin/pkill -TERM -v -u 0,1,15", 1, 5);
3564         (void) fork_with_timeout("/usr/bin/pkill -KILL -v -u 0,1,15", 1, 5);
3565 }
3566 
3567 static void
3568 do_uadmin(void)
3569 {
3570         const char * const resetting = "/etc/svc/volatile/resetting";
3571         int fd;
3572         struct statvfs vfs;
3573         time_t now;
3574         struct tm nowtm;
3575         char down_buf[256], time_buf[256];
3576         uintptr_t mdep;
3577 #if defined(__i386)
3578         grub_boot_args_t fbarg;
3579 #endif  /* __i386 */
3580 
3581         mdep = NULL;
3582         fd = creat(resetting, 0777);
3583         if (fd >= 0)
3584                 startd_close(fd);
3585         else
3586                 uu_warn("Could not create \"%s\"", resetting);
3587 
3588         /* Kill dhcpagent if we're not using nfs for root */
3589         if ((statvfs("/", &vfs) == 0) &&
3590             (strncmp(vfs.f_basetype, "nfs", sizeof ("nfs") - 1) != 0))
3591                 fork_with_timeout("/usr/bin/pkill -x -u 0 dhcpagent", 0, 5);
3592 
3593         /*
3594          * Call sync(2) now, before we kill off user processes.  This takes
3595          * advantage of the several seconds of pause we have before the
3596          * killalls are done.  Time we can make good use of to get pages
3597          * moving out to disk.
3598          *
3599          * Inside non-global zones, we don't bother, and it's better not to
3600          * anyway, since sync(2) can have system-wide impact.
3601          */
3602         if (getzoneid() == 0)
3603                 sync();
3604 
3605         kill_user_procs();
3606 
3607         /*
3608          * Note that this must come after the killing of user procs, since
3609          * killall relies on utmpx, and this command affects the contents of
3610          * said file.
3611          */
3612         if (access("/usr/lib/acct/closewtmp", X_OK) == 0)
3613                 fork_with_timeout("/usr/lib/acct/closewtmp", 0, 5);
3614 
3615         /*
3616          * For patches which may be installed as the system is shutting
3617          * down, we need to ensure, one more time, that the boot archive
3618          * really is up to date.
3619          */
3620         if (getzoneid() == 0 && access("/usr/sbin/bootadm", X_OK) == 0)
3621                 fork_with_timeout("/usr/sbin/bootadm -ea update_all", 0, 3600);
3622 
3623         /*
3624          * Right now, fast reboot is supported only on i386.
3625          * scf_is_fastboot_default() should take care of it.
3626          * If somehow we got there on unsupported platform -
3627          * print warning and fall back to regular reboot.
3628          */
3629         if (halting == AD_FASTREBOOT) {
3630 #if defined(__i386)
3631                 int rc;
3632 
3633                 if ((rc = grub_get_boot_args(&fbarg, NULL,
3634                     GRUB_ENTRY_DEFAULT)) == 0) {
3635                         mdep = (uintptr_t)&fbarg.gba_bootargs;
3636                 } else {
3637                         /*
3638                          * Failed to read GRUB menu, fall back to normal reboot
3639                          */
3640                         halting = AD_BOOT;
3641                         uu_warn("Failed to process GRUB menu entry "
3642                             "for fast reboot.\n\t%s\n"
3643                             "Falling back to regular reboot.\n",
3644                             grub_strerror(rc));
3645                 }
3646 #else   /* __i386 */
3647                 halting = AD_BOOT;
3648                 uu_warn("Fast reboot configured, but not supported by "
3649                     "this ISA\n");
3650 #endif  /* __i386 */
3651         }
3652 
3653         fork_with_timeout("/sbin/umountall -l", 0, 5);
3654         fork_with_timeout("/sbin/umount /tmp /var/adm /var/run /var "
3655             ">/dev/null 2>&1", 0, 5);
3656 
3657         /*
3658          * Try to get to consistency for whatever UFS filesystems are left.
3659          * This is pretty expensive, so we save it for the end in the hopes of
3660          * minimizing what it must do.  The other option would be to start in
3661          * parallel with the killall's, but lockfs tends to throw out much more
3662          * than is needed, and so subsequent commands (like umountall) take a
3663          * long time to get going again.
3664          *
3665          * Inside of zones, we don't bother, since we're not about to terminate
3666          * the whole OS instance.
3667          *
3668          * On systems using only ZFS, this call to lockfs -fa is a no-op.
3669          */
3670         if (getzoneid() == 0) {
3671                 if (access("/usr/sbin/lockfs", X_OK) == 0)
3672                         fork_with_timeout("/usr/sbin/lockfs -fa", 0, 30);
3673 
3674                 sync(); /* once more, with feeling */
3675         }
3676 
3677         fork_with_timeout("/sbin/umount /usr >/dev/null 2>&1", 0, 5);
3678 
3679         /*
3680          * Construct and emit the last words from userland:
3681          * "<timestamp> The system is down.  Shutdown took <N> seconds."
3682          *
3683          * Normally we'd use syslog, but with /var and other things
3684          * potentially gone, try to minimize the external dependencies.
3685          */
3686         now = time(NULL);
3687         (void) localtime_r(&now, &nowtm);
3688 
3689         if (strftime(down_buf, sizeof (down_buf),
3690             "%b %e %T The system is down.", &nowtm) == 0) {
3691                 (void) strlcpy(down_buf, "The system is down.",
3692                     sizeof (down_buf));
3693         }
3694 
3695         if (halting_time != 0 && halting_time <= now) {
3696                 (void) snprintf(time_buf, sizeof (time_buf),
3697                     "  Shutdown took %lu seconds.", now - halting_time);
3698         } else {
3699                 time_buf[0] = '\0';
3700         }
3701         (void) printf("%s%s\n", down_buf, time_buf);
3702 
3703         (void) uadmin(A_SHUTDOWN, halting, mdep);
3704         uu_warn("uadmin() failed");
3705 
3706 #if defined(__i386)
3707         /* uadmin fail, cleanup grub_boot_args */
3708         if (halting == AD_FASTREBOOT)
3709                 grub_cleanup_boot_args(&fbarg);
3710 #endif  /* __i386 */
3711 
3712         if (remove(resetting) != 0 && errno != ENOENT)
3713                 uu_warn("Could not remove \"%s\"", resetting);
3714 }
3715 
3716 /*
3717  * If any of the up_svcs[] are online or satisfiable, return true.  If they are
3718  * all missing, disabled, in maintenance, or unsatisfiable, return false.
3719  */
3720 boolean_t
3721 can_come_up(void)
3722 {
3723         int i;
3724 
3725         assert(MUTEX_HELD(&dgraph_lock));
3726 
3727         /*
3728          * If we are booting to single user (boot -s),
3729          * SCF_MILESTONE_SINGLE_USER is needed to come up because startd
3730          * spawns sulogin after single-user is online (see specials.c).
3731          */
3732         i = (booting_to_single_user ? 0 : 1);
3733 
3734         for (; up_svcs[i] != NULL; ++i) {
3735                 if (up_svcs_p[i] == NULL) {
3736                         up_svcs_p[i] = vertex_get_by_name(up_svcs[i]);
3737 
3738                         if (up_svcs_p[i] == NULL)
3739                                 continue;
3740                 }
3741 
3742                 /*
3743                  * Ignore unconfigured services (the ones that have been
3744                  * mentioned in a dependency from other services, but do
3745                  * not exist in the repository).  Services which exist
3746                  * in the repository but don't have general/enabled
3747                  * property will be also ignored.
3748                  */
3749                 if (!(up_svcs_p[i]->gv_flags & GV_CONFIGURED))
3750                         continue;
3751 
3752                 switch (up_svcs_p[i]->gv_state) {
3753                 case RESTARTER_STATE_ONLINE:
3754                 case RESTARTER_STATE_DEGRADED:
3755                         /*
3756                          * Deactivate verbose boot once a login service has been
3757                          * reached.
3758                          */
3759                         st->st_log_login_reached = 1;
3760                         /*FALLTHROUGH*/
3761                 case RESTARTER_STATE_UNINIT:
3762                         return (B_TRUE);
3763 
3764                 case RESTARTER_STATE_OFFLINE:
3765                         if (instance_satisfied(up_svcs_p[i], B_TRUE) != -1)
3766                                 return (B_TRUE);
3767                         log_framework(LOG_DEBUG,
3768                             "can_come_up(): %s is unsatisfiable.\n",
3769                             up_svcs_p[i]->gv_name);
3770                         continue;
3771 
3772                 case RESTARTER_STATE_DISABLED:
3773                 case RESTARTER_STATE_MAINT:
3774                         log_framework(LOG_DEBUG,
3775                             "can_come_up(): %s is in state %s.\n",
3776                             up_svcs_p[i]->gv_name,
3777                             instance_state_str[up_svcs_p[i]->gv_state]);
3778                         continue;
3779 
3780                 default:
3781 #ifndef NDEBUG
3782                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
3783                             __FILE__, __LINE__, up_svcs_p[i]->gv_state);
3784 #endif
3785                         abort();
3786                 }
3787         }
3788 
3789         /*
3790          * In the seed repository, console-login is unsatisfiable because
3791          * services are missing.  To behave correctly in that case we don't want
3792          * to return false until manifest-import is online.
3793          */
3794 
3795         if (manifest_import_p == NULL) {
3796                 manifest_import_p = vertex_get_by_name(manifest_import);
3797 
3798                 if (manifest_import_p == NULL)
3799                         return (B_FALSE);
3800         }
3801 
3802         switch (manifest_import_p->gv_state) {
3803         case RESTARTER_STATE_ONLINE:
3804         case RESTARTER_STATE_DEGRADED:
3805         case RESTARTER_STATE_DISABLED:
3806         case RESTARTER_STATE_MAINT:
3807                 break;
3808 
3809         case RESTARTER_STATE_OFFLINE:
3810                 if (instance_satisfied(manifest_import_p, B_TRUE) == -1)
3811                         break;
3812                 /* FALLTHROUGH */
3813 
3814         case RESTARTER_STATE_UNINIT:
3815                 return (B_TRUE);
3816         }
3817 
3818         return (B_FALSE);
3819 }
3820 
3821 /*
3822  * Runs sulogin.  Returns
3823  *   0 - success
3824  *   EALREADY - sulogin is already running
3825  *   EBUSY - console-login is running
3826  */
3827 static int
3828 run_sulogin(const char *msg)
3829 {
3830         graph_vertex_t *v;
3831 
3832         assert(MUTEX_HELD(&dgraph_lock));
3833 
3834         if (sulogin_running)
3835                 return (EALREADY);
3836 
3837         v = vertex_get_by_name(console_login_fmri);
3838         if (v != NULL && inst_running(v))
3839                 return (EBUSY);
3840 
3841         sulogin_running = B_TRUE;
3842 
3843         MUTEX_UNLOCK(&dgraph_lock);
3844 
3845         fork_sulogin(B_FALSE, msg);
3846 
3847         MUTEX_LOCK(&dgraph_lock);
3848 
3849         sulogin_running = B_FALSE;
3850 
3851         if (console_login_ready) {
3852                 v = vertex_get_by_name(console_login_fmri);
3853 
3854                 if (v != NULL && v->gv_state == RESTARTER_STATE_OFFLINE) {
3855                         if (v->gv_start_f == NULL)
3856                                 vertex_send_event(v,
3857                                     RESTARTER_EVENT_TYPE_START);
3858                         else
3859                                 v->gv_start_f(v);
3860                 }
3861 
3862                 console_login_ready = B_FALSE;
3863         }
3864 
3865         return (0);
3866 }
3867 
3868 /*
3869  * The sulogin thread runs sulogin while can_come_up() is false.  run_sulogin()
3870  * keeps sulogin from stepping on console-login's toes.
3871  */
3872 /* ARGSUSED */
3873 static void *
3874 sulogin_thread(void *unused)
3875 {
3876         MUTEX_LOCK(&dgraph_lock);
3877 
3878         assert(sulogin_thread_running);
3879 
3880         do {
3881                 (void) run_sulogin("Console login service(s) cannot run\n");
3882         } while (!can_come_up());
3883 
3884         sulogin_thread_running = B_FALSE;
3885         MUTEX_UNLOCK(&dgraph_lock);
3886 
3887         return (NULL);
3888 }
3889 
3890 /* ARGSUSED */
3891 void *
3892 single_user_thread(void *unused)
3893 {
3894         uint_t left;
3895         scf_handle_t *h;
3896         scf_instance_t *inst;
3897         scf_property_t *prop;
3898         scf_value_t *val;
3899         const char *msg;
3900         char *buf;
3901         int r;
3902 
3903         MUTEX_LOCK(&single_user_thread_lock);
3904         single_user_thread_count++;
3905 
3906         if (!booting_to_single_user)
3907                 kill_user_procs();
3908 
3909         if (go_single_user_mode || booting_to_single_user) {
3910                 msg = "SINGLE USER MODE\n";
3911         } else {
3912                 assert(go_to_level1);
3913 
3914                 fork_rc_script('1', "start", B_TRUE);
3915 
3916                 uu_warn("The system is ready for administration.\n");
3917 
3918                 msg = "";
3919         }
3920 
3921         MUTEX_UNLOCK(&single_user_thread_lock);
3922 
3923         for (;;) {
3924                 MUTEX_LOCK(&dgraph_lock);
3925                 r = run_sulogin(msg);
3926                 MUTEX_UNLOCK(&dgraph_lock);
3927                 if (r == 0)
3928                         break;
3929 
3930                 assert(r == EALREADY || r == EBUSY);
3931 
3932                 left = 3;
3933                 while (left > 0)
3934                         left = sleep(left);
3935         }
3936 
3937         MUTEX_LOCK(&single_user_thread_lock);
3938 
3939         /*
3940          * If another single user thread has started, let it finish changing
3941          * the run level.
3942          */
3943         if (single_user_thread_count > 1) {
3944                 single_user_thread_count--;
3945                 MUTEX_UNLOCK(&single_user_thread_lock);
3946                 return (NULL);
3947         }
3948 
3949         h = libscf_handle_create_bound_loop();
3950         inst = scf_instance_create(h);
3951         prop = safe_scf_property_create(h);
3952         val = safe_scf_value_create(h);
3953         buf = startd_alloc(max_scf_fmri_size);
3954 
3955 lookup:
3956         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
3957             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
3958                 switch (scf_error()) {
3959                 case SCF_ERROR_NOT_FOUND:
3960                         r = libscf_create_self(h);
3961                         if (r == 0)
3962                                 goto lookup;
3963                         assert(r == ECONNABORTED);
3964                         /* FALLTHROUGH */
3965 
3966                 case SCF_ERROR_CONNECTION_BROKEN:
3967                         libscf_handle_rebind(h);
3968                         goto lookup;
3969 
3970                 case SCF_ERROR_INVALID_ARGUMENT:
3971                 case SCF_ERROR_CONSTRAINT_VIOLATED:
3972                 case SCF_ERROR_NOT_BOUND:
3973                 case SCF_ERROR_HANDLE_MISMATCH:
3974                 default:
3975                         bad_error("scf_handle_decode_fmri", scf_error());
3976                 }
3977         }
3978 
3979         MUTEX_LOCK(&dgraph_lock);
3980 
3981         r = scf_instance_delete_prop(inst, SCF_PG_OPTIONS_OVR,
3982             SCF_PROPERTY_MILESTONE);
3983         switch (r) {
3984         case 0:
3985         case ECANCELED:
3986                 break;
3987 
3988         case ECONNABORTED:
3989                 MUTEX_UNLOCK(&dgraph_lock);
3990                 libscf_handle_rebind(h);
3991                 goto lookup;
3992 
3993         case EPERM:
3994         case EACCES:
3995         case EROFS:
3996                 log_error(LOG_WARNING, "Could not clear temporary milestone: "
3997                     "%s.\n", strerror(r));
3998                 break;
3999 
4000         default:
4001                 bad_error("scf_instance_delete_prop", r);
4002         }
4003 
4004         MUTEX_UNLOCK(&dgraph_lock);
4005 
4006         r = libscf_get_milestone(inst, prop, val, buf, max_scf_fmri_size);
4007         switch (r) {
4008         case ECANCELED:
4009         case ENOENT:
4010         case EINVAL:
4011                 (void) strcpy(buf, "all");
4012                 /* FALLTHROUGH */
4013 
4014         case 0:
4015                 uu_warn("Returning to milestone %s.\n", buf);
4016                 break;
4017 
4018         case ECONNABORTED:
4019                 libscf_handle_rebind(h);
4020                 goto lookup;
4021 
4022         default:
4023                 bad_error("libscf_get_milestone", r);
4024         }
4025 
4026         r = dgraph_set_milestone(buf, h, B_FALSE);
4027         switch (r) {
4028         case 0:
4029         case ECONNRESET:
4030         case EALREADY:
4031         case EINVAL:
4032         case ENOENT:
4033                 break;
4034 
4035         default:
4036                 bad_error("dgraph_set_milestone", r);
4037         }
4038 
4039         /*
4040          * See graph_runlevel_changed().
4041          */
4042         MUTEX_LOCK(&dgraph_lock);
4043         utmpx_set_runlevel(target_milestone_as_runlevel(), 'S', B_TRUE);
4044         MUTEX_UNLOCK(&dgraph_lock);
4045 
4046         startd_free(buf, max_scf_fmri_size);
4047         scf_value_destroy(val);
4048         scf_property_destroy(prop);
4049         scf_instance_destroy(inst);
4050         scf_handle_destroy(h);
4051 
4052         /*
4053          * We'll give ourselves 3 seconds to respond to all of the enablings
4054          * that setting the milestone should have created before checking
4055          * whether to run sulogin.
4056          */
4057         left = 3;
4058         while (left > 0)
4059                 left = sleep(left);
4060 
4061         MUTEX_LOCK(&dgraph_lock);
4062         /*
4063          * Clearing these variables will allow the sulogin thread to run.  We
4064          * check here in case there aren't any more state updates anytime soon.
4065          */
4066         go_to_level1 = go_single_user_mode = booting_to_single_user = B_FALSE;
4067         if (!sulogin_thread_running && !can_come_up()) {
4068                 (void) startd_thread_create(sulogin_thread, NULL);
4069                 sulogin_thread_running = B_TRUE;
4070         }
4071         MUTEX_UNLOCK(&dgraph_lock);
4072         single_user_thread_count--;
4073         MUTEX_UNLOCK(&single_user_thread_lock);
4074         return (NULL);
4075 }
4076 
4077 
4078 /*
4079  * Dependency graph operations API.  These are handle-independent thread-safe
4080  * graph manipulation functions which are the entry points for the event
4081  * threads below.
4082  */
4083 
4084 /*
4085  * If a configured vertex exists for inst_fmri, return EEXIST.  If no vertex
4086  * exists for inst_fmri, add one.  Then fetch the restarter from inst, make
4087  * this vertex dependent on it, and send _ADD_INSTANCE to the restarter.
4088  * Fetch whether the instance should be enabled from inst and send _ENABLE or
4089  * _DISABLE as appropriate.  Finally rummage through inst's dependency
4090  * property groups and add vertices and edges as appropriate.  If anything
4091  * goes wrong after sending _ADD_INSTANCE, send _ADMIN_MAINT_ON to put the
4092  * instance in maintenance.  Don't send _START or _STOP until we get a state
4093  * update in case we're being restarted and the service is already running.
4094  *
4095  * To support booting to a milestone, we must also make sure all dependencies
4096  * encountered are configured, if they exist in the repository.
4097  *
4098  * Returns 0 on success, ECONNABORTED on repository disconnection, EINVAL if
4099  * inst_fmri is an invalid (or not canonical) FMRI, ECANCELED if inst is
4100  * deleted, or EEXIST if a configured vertex for inst_fmri already exists.
4101  */
4102 int
4103 dgraph_add_instance(const char *inst_fmri, scf_instance_t *inst,
4104     boolean_t lock_graph)
4105 {
4106         graph_vertex_t *v;
4107         int err;
4108 
4109         if (strcmp(inst_fmri, SCF_SERVICE_STARTD) == 0)
4110                 return (0);
4111 
4112         /* Check for a vertex for inst_fmri. */
4113         if (lock_graph) {
4114                 MUTEX_LOCK(&dgraph_lock);
4115         } else {
4116                 assert(MUTEX_HELD(&dgraph_lock));
4117         }
4118 
4119         v = vertex_get_by_name(inst_fmri);
4120 
4121         if (v != NULL) {
4122                 assert(v->gv_type == GVT_INST);
4123 
4124                 if (v->gv_flags & GV_CONFIGURED) {
4125                         if (lock_graph)
4126                                 MUTEX_UNLOCK(&dgraph_lock);
4127                         return (EEXIST);
4128                 }
4129         } else {
4130                 /* Add the vertex. */
4131                 err = graph_insert_vertex_unconfigured(inst_fmri, GVT_INST, 0,
4132                     RERR_NONE, &v);
4133                 if (err != 0) {
4134                         assert(err == EINVAL);
4135                         if (lock_graph)
4136                                 MUTEX_UNLOCK(&dgraph_lock);
4137                         return (EINVAL);
4138                 }
4139         }
4140 
4141         err = configure_vertex(v, inst);
4142 
4143         if (lock_graph)
4144                 MUTEX_UNLOCK(&dgraph_lock);
4145 
4146         return (err);
4147 }
4148 
4149 /*
4150  * Locate the vertex for this property group's instance.  If it doesn't exist
4151  * or is unconfigured, call dgraph_add_instance() & return.  Otherwise fetch
4152  * the restarter for the instance, and if it has changed, send
4153  * _REMOVE_INSTANCE to the old restarter, remove the dependency, make sure the
4154  * new restarter has a vertex, add a new dependency, and send _ADD_INSTANCE to
4155  * the new restarter.  Then fetch whether the instance should be enabled, and
4156  * if it is different from what we had, or if we changed the restarter, send
4157  * the appropriate _ENABLE or _DISABLE command.
4158  *
4159  * Returns 0 on success, ENOTSUP if the pg's parent is not an instance,
4160  * ECONNABORTED on repository disconnection, ECANCELED if the instance is
4161  * deleted, or -1 if the instance's general property group is deleted or if
4162  * its enabled property is misconfigured.
4163  */
4164 static int
4165 dgraph_update_general(scf_propertygroup_t *pg)
4166 {
4167         scf_handle_t *h;
4168         scf_instance_t *inst;
4169         char *fmri;
4170         char *restarter_fmri;
4171         graph_vertex_t *v;
4172         int err;
4173         int enabled, enabled_ovr;
4174         int oldflags;
4175 
4176         /* Find the vertex for this service */
4177         h = scf_pg_handle(pg);
4178 
4179         inst = safe_scf_instance_create(h);
4180 
4181         if (scf_pg_get_parent_instance(pg, inst) != 0) {
4182                 switch (scf_error()) {
4183                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4184                         return (ENOTSUP);
4185 
4186                 case SCF_ERROR_CONNECTION_BROKEN:
4187                 default:
4188                         return (ECONNABORTED);
4189 
4190                 case SCF_ERROR_DELETED:
4191                         return (0);
4192 
4193                 case SCF_ERROR_NOT_SET:
4194                         bad_error("scf_pg_get_parent_instance", scf_error());
4195                 }
4196         }
4197 
4198         err = libscf_instance_get_fmri(inst, &fmri);
4199         switch (err) {
4200         case 0:
4201                 break;
4202 
4203         case ECONNABORTED:
4204                 scf_instance_destroy(inst);
4205                 return (ECONNABORTED);
4206 
4207         case ECANCELED:
4208                 scf_instance_destroy(inst);
4209                 return (0);
4210 
4211         default:
4212                 bad_error("libscf_instance_get_fmri", err);
4213         }
4214 
4215         log_framework(LOG_DEBUG,
4216             "Graph engine: Reloading general properties for %s.\n", fmri);
4217 
4218         MUTEX_LOCK(&dgraph_lock);
4219 
4220         v = vertex_get_by_name(fmri);
4221         if (v == NULL || !(v->gv_flags & GV_CONFIGURED)) {
4222                 /* Will get the up-to-date properties. */
4223                 MUTEX_UNLOCK(&dgraph_lock);
4224                 err = dgraph_add_instance(fmri, inst, B_TRUE);
4225                 startd_free(fmri, max_scf_fmri_size);
4226                 scf_instance_destroy(inst);
4227                 return (err == ECANCELED ? 0 : err);
4228         }
4229 
4230         /* Read enabled & restarter from repository. */
4231         restarter_fmri = startd_alloc(max_scf_value_size);
4232         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
4233             &enabled_ovr, &restarter_fmri);
4234         if (err != 0 || enabled == -1) {
4235                 MUTEX_UNLOCK(&dgraph_lock);
4236                 scf_instance_destroy(inst);
4237                 startd_free(fmri, max_scf_fmri_size);
4238 
4239                 switch (err) {
4240                 case ENOENT:
4241                 case 0:
4242                         startd_free(restarter_fmri, max_scf_value_size);
4243                         return (-1);
4244 
4245                 case ECONNABORTED:
4246                 case ECANCELED:
4247                         startd_free(restarter_fmri, max_scf_value_size);
4248                         return (err);
4249 
4250                 default:
4251                         bad_error("libscf_get_basic_instance_data", err);
4252                 }
4253         }
4254 
4255         oldflags = v->gv_flags;
4256         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
4257             (enabled ? GV_ENBLD_NOOVR : 0);
4258 
4259         if (enabled_ovr != -1)
4260                 enabled = enabled_ovr;
4261 
4262         /*
4263          * If GV_ENBLD_NOOVR has changed, then we need to re-evaluate the
4264          * subgraph.
4265          */
4266         if (milestone > MILESTONE_NONE && v->gv_flags != oldflags)
4267                 (void) eval_subgraph(v, h);
4268 
4269         scf_instance_destroy(inst);
4270 
4271         /* Ignore restarter change for now. */
4272 
4273         startd_free(restarter_fmri, max_scf_value_size);
4274         startd_free(fmri, max_scf_fmri_size);
4275 
4276         /*
4277          * Always send _ENABLE or _DISABLE.  We could avoid this if the
4278          * restarter didn't change and the enabled value didn't change, but
4279          * that's not easy to check and improbable anyway, so we'll just do
4280          * this.
4281          */
4282         graph_enable_by_vertex(v, enabled, 1);
4283 
4284         MUTEX_UNLOCK(&dgraph_lock);
4285 
4286         return (0);
4287 }
4288 
4289 /*
4290  * Delete all of the property group dependencies of v, update inst's running
4291  * snapshot, and add the dependencies in the new snapshot.  If any of the new
4292  * dependencies would create a cycle, send _ADMIN_MAINT_ON.  Otherwise
4293  * reevaluate v's dependencies, send _START or _STOP as appropriate, and do
4294  * the same for v's dependents.
4295  *
4296  * Returns
4297  *   0 - success
4298  *   ECONNABORTED - repository connection broken
4299  *   ECANCELED - inst was deleted
4300  *   EINVAL - inst is invalid (e.g., missing general/enabled)
4301  *   -1 - libscf_snapshots_refresh() failed
4302  */
4303 static int
4304 dgraph_refresh_instance(graph_vertex_t *v, scf_instance_t *inst)
4305 {
4306         int r;
4307         int enabled;
4308         int32_t tset;
4309 
4310         assert(MUTEX_HELD(&dgraph_lock));
4311         assert(v->gv_type == GVT_INST);
4312 
4313         /* Only refresh services with valid general/enabled properties. */
4314         r = libscf_get_basic_instance_data(scf_instance_handle(inst), inst,
4315             v->gv_name, &enabled, NULL, NULL);
4316         switch (r) {
4317         case 0:
4318                 break;
4319 
4320         case ECONNABORTED:
4321         case ECANCELED:
4322                 return (r);
4323 
4324         case ENOENT:
4325                 log_framework(LOG_DEBUG,
4326                     "Ignoring %s because it has no general property group.\n",
4327                     v->gv_name);
4328                 return (EINVAL);
4329 
4330         default:
4331                 bad_error("libscf_get_basic_instance_data", r);
4332         }
4333 
4334         if ((tset = libscf_get_stn_tset(inst)) == -1) {
4335                 log_framework(LOG_WARNING,
4336                     "Failed to get notification parameters for %s: %s\n",
4337                     v->gv_name, scf_strerror(scf_error()));
4338                 tset = 0;
4339         }
4340         v->gv_stn_tset = tset;
4341         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
4342                 stn_global = tset;
4343 
4344         if (enabled == -1)
4345                 return (EINVAL);
4346 
4347         r = libscf_snapshots_refresh(inst, v->gv_name);
4348         if (r != 0) {
4349                 if (r != -1)
4350                         bad_error("libscf_snapshots_refresh", r);
4351 
4352                 /* error logged */
4353                 return (r);
4354         }
4355 
4356         r = refresh_vertex(v, inst);
4357         if (r != 0 && r != ECONNABORTED)
4358                 bad_error("refresh_vertex", r);
4359         return (r);
4360 }
4361 
4362 /*
4363  * Returns true only if none of this service's dependents are 'up' -- online
4364  * or degraded (offline is considered down in this situation). This function
4365  * is somehow similar to is_nonsubgraph_leaf() but works on subtrees.
4366  */
4367 boolean_t
4368 insubtree_dependents_down(graph_vertex_t *v)
4369 {
4370         graph_vertex_t *vv;
4371         graph_edge_t *e;
4372 
4373         assert(MUTEX_HELD(&dgraph_lock));
4374 
4375         for (e = uu_list_first(v->gv_dependents); e != NULL;
4376             e = uu_list_next(v->gv_dependents, e)) {
4377                 vv = e->ge_vertex;
4378                 if (vv->gv_type == GVT_INST) {
4379                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4380                                 continue;
4381 
4382                         if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4383                                 return (B_FALSE);
4384 
4385                         if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4386                             (vv->gv_state == RESTARTER_STATE_DEGRADED))
4387                                 return (B_FALSE);
4388                 } else {
4389                         /*
4390                          * Skip all excluded dependencies and decide whether
4391                          * decide whether to offline the service based on the
4392                          * restart_on attribute.
4393                          */
4394                         if (is_depgrp_bypassed(vv))
4395                                 continue;
4396 
4397                         /*
4398                          * For dependency groups or service vertices, keep
4399                          * traversing to see if instances are running.
4400                          */
4401                         if (insubtree_dependents_down(vv) == B_FALSE)
4402                                 return (B_FALSE);
4403                 }
4404         }
4405 
4406         return (B_TRUE);
4407 }
4408 
4409 /*
4410  * Returns true only if none of this service's dependents are 'up' -- online,
4411  * degraded, or offline.
4412  */
4413 static int
4414 is_nonsubgraph_leaf(graph_vertex_t *v)
4415 {
4416         graph_vertex_t *vv;
4417         graph_edge_t *e;
4418 
4419         assert(MUTEX_HELD(&dgraph_lock));
4420 
4421         for (e = uu_list_first(v->gv_dependents);
4422             e != NULL;
4423             e = uu_list_next(v->gv_dependents, e)) {
4424 
4425                 vv = e->ge_vertex;
4426                 if (vv->gv_type == GVT_INST) {
4427                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4428                                 continue;
4429 
4430                         if (vv->gv_flags & GV_INSUBGRAPH)
4431                                 continue;
4432 
4433                         if (up_state(vv->gv_state))
4434                                 return (0);
4435                 } else {
4436                         /*
4437                          * For dependency group or service vertices, keep
4438                          * traversing to see if instances are running.
4439                          *
4440                          * We should skip exclude_all dependencies otherwise
4441                          * the vertex will never be considered as a leaf
4442                          * if the dependent is offline. The main reason for
4443                          * this is that disable_nonsubgraph_leaves() skips
4444                          * exclusion dependencies.
4445                          */
4446                         if (vv->gv_type == GVT_GROUP &&
4447                             vv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4448                                 continue;
4449 
4450                         if (!is_nonsubgraph_leaf(vv))
4451                                 return (0);
4452                 }
4453         }
4454 
4455         return (1);
4456 }
4457 
4458 /*
4459  * Disable v temporarily.  Attempt to do this by setting its enabled override
4460  * property in the repository.  If that fails, send a _DISABLE command.
4461  * Returns 0 on success and ECONNABORTED if the repository connection is
4462  * broken.
4463  */
4464 static int
4465 disable_service_temporarily(graph_vertex_t *v, scf_handle_t *h)
4466 {
4467         const char * const emsg = "Could not temporarily disable %s because "
4468             "%s.  Will stop service anyways.  Repository status for the "
4469             "service may be inaccurate.\n";
4470         const char * const emsg_cbroken =
4471             "the repository connection was broken";
4472 
4473         scf_instance_t *inst;
4474         int r;
4475 
4476         inst = scf_instance_create(h);
4477         if (inst == NULL) {
4478                 char buf[100];
4479 
4480                 (void) snprintf(buf, sizeof (buf),
4481                     "scf_instance_create() failed (%s)",
4482                     scf_strerror(scf_error()));
4483                 log_error(LOG_WARNING, emsg, v->gv_name, buf);
4484 
4485                 graph_enable_by_vertex(v, 0, 0);
4486                 return (0);
4487         }
4488 
4489         r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
4490             NULL, NULL, SCF_DECODE_FMRI_EXACT);
4491         if (r != 0) {
4492                 switch (scf_error()) {
4493                 case SCF_ERROR_CONNECTION_BROKEN:
4494                         log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4495                         graph_enable_by_vertex(v, 0, 0);
4496                         return (ECONNABORTED);
4497 
4498                 case SCF_ERROR_NOT_FOUND:
4499                         return (0);
4500 
4501                 case SCF_ERROR_HANDLE_MISMATCH:
4502                 case SCF_ERROR_INVALID_ARGUMENT:
4503                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4504                 case SCF_ERROR_NOT_BOUND:
4505                 default:
4506                         bad_error("scf_handle_decode_fmri",
4507                             scf_error());
4508                 }
4509         }
4510 
4511         r = libscf_set_enable_ovr(inst, 0);
4512         switch (r) {
4513         case 0:
4514                 scf_instance_destroy(inst);
4515                 return (0);
4516 
4517         case ECANCELED:
4518                 scf_instance_destroy(inst);
4519                 return (0);
4520 
4521         case ECONNABORTED:
4522                 log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4523                 graph_enable_by_vertex(v, 0, 0);
4524                 return (ECONNABORTED);
4525 
4526         case EPERM:
4527                 log_error(LOG_WARNING, emsg, v->gv_name,
4528                     "the repository denied permission");
4529                 graph_enable_by_vertex(v, 0, 0);
4530                 return (0);
4531 
4532         case EROFS:
4533                 log_error(LOG_WARNING, emsg, v->gv_name,
4534                     "the repository is read-only");
4535                 graph_enable_by_vertex(v, 0, 0);
4536                 return (0);
4537 
4538         default:
4539                 bad_error("libscf_set_enable_ovr", r);
4540                 /* NOTREACHED */
4541         }
4542 }
4543 
4544 /*
4545  * Of the transitive instance dependencies of v, offline those which are
4546  * in the subtree and which are leaves (i.e., have no dependents which are
4547  * "up").
4548  */
4549 void
4550 offline_subtree_leaves(graph_vertex_t *v, void *arg)
4551 {
4552         assert(MUTEX_HELD(&dgraph_lock));
4553 
4554         /* If v has up dependents, try to bring them down first. */
4555         if (insubtree_dependents_down(v) == B_FALSE) {
4556                 graph_walk_dependents(v, offline_subtree_leaves, arg);
4557 
4558                 /* If we couldn't bring them down, return. */
4559                 if (insubtree_dependents_down(v) == B_FALSE)
4560                         return;
4561         }
4562 
4563         /* If v isn't an instance, recurse on its dependencies. */
4564         if (v->gv_type != GVT_INST) {
4565                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4566                 return;
4567         }
4568 
4569         /*
4570          * If v is not in the subtree, so should all of its dependencies,
4571          * so do nothing.
4572          */
4573         if ((v->gv_flags & GV_TOOFFLINE) == 0)
4574                 return;
4575 
4576         /* If v isn't a leaf because it's already down, recurse. */
4577         if (!up_state(v->gv_state)) {
4578                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4579                 return;
4580         }
4581 
4582         /* if v is a leaf, offline it or disable it if it's the last one */
4583         if (insubtree_dependents_down(v) == B_TRUE) {
4584                 if (v->gv_flags & GV_TODISABLE)
4585                         vertex_send_event(v,
4586                             RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
4587                 else
4588                         offline_vertex(v);
4589         }
4590 }
4591 
4592 void
4593 graph_offline_subtree_leaves(graph_vertex_t *v, void *h)
4594 {
4595         graph_walk_dependencies(v, offline_subtree_leaves, (void *)h);
4596 }
4597 
4598 
4599 /*
4600  * Of the transitive instance dependencies of v, disable those which are not
4601  * in the subgraph and which are leaves (i.e., have no dependents which are
4602  * "up").
4603  */
4604 static void
4605 disable_nonsubgraph_leaves(graph_vertex_t *v, void *arg)
4606 {
4607         assert(MUTEX_HELD(&dgraph_lock));
4608 
4609         /*
4610          * We must skip exclusion dependencies because they are allowed to
4611          * complete dependency cycles.  This is correct because A's exclusion
4612          * dependency on B doesn't bear on the order in which they should be
4613          * stopped.  Indeed, the exclusion dependency should guarantee that
4614          * they are never online at the same time.
4615          */
4616         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4617                 return;
4618 
4619         /* If v isn't an instance, recurse on its dependencies. */
4620         if (v->gv_type != GVT_INST)
4621                 goto recurse;
4622 
4623         if ((v->gv_flags & GV_CONFIGURED) == 0)
4624                 /*
4625                  * Unconfigured instances should have no dependencies, but in
4626                  * case they ever get them,
4627                  */
4628                 goto recurse;
4629 
4630         /*
4631          * If v is in the subgraph, so should all of its dependencies, so do
4632          * nothing.
4633          */
4634         if (v->gv_flags & GV_INSUBGRAPH)
4635                 return;
4636 
4637         /* If v isn't a leaf because it's already down, recurse. */
4638         if (!up_state(v->gv_state))
4639                 goto recurse;
4640 
4641         /* If v is disabled but not down yet, be patient. */
4642         if ((v->gv_flags & GV_ENABLED) == 0)
4643                 return;
4644 
4645         /* If v is a leaf, disable it. */
4646         if (is_nonsubgraph_leaf(v))
4647                 (void) disable_service_temporarily(v, (scf_handle_t *)arg);
4648 
4649         return;
4650 
4651 recurse:
4652         graph_walk_dependencies(v, disable_nonsubgraph_leaves, arg);
4653 }
4654 
4655 static int
4656 stn_restarter_state(restarter_instance_state_t rstate)
4657 {
4658         static const struct statemap {
4659                 restarter_instance_state_t restarter_state;
4660                 int scf_state;
4661         } map[] = {
4662                 { RESTARTER_STATE_UNINIT, SCF_STATE_UNINIT },
4663                 { RESTARTER_STATE_MAINT, SCF_STATE_MAINT },
4664                 { RESTARTER_STATE_OFFLINE, SCF_STATE_OFFLINE },
4665                 { RESTARTER_STATE_DISABLED, SCF_STATE_DISABLED },
4666                 { RESTARTER_STATE_ONLINE, SCF_STATE_ONLINE },
4667                 { RESTARTER_STATE_DEGRADED, SCF_STATE_DEGRADED }
4668         };
4669 
4670         int i;
4671 
4672         for (i = 0; i < sizeof (map) / sizeof (map[0]); i++) {
4673                 if (rstate == map[i].restarter_state)
4674                         return (map[i].scf_state);
4675         }
4676 
4677         return (-1);
4678 }
4679 
4680 /*
4681  * State transition counters
4682  * Not incremented atomically - indicative only
4683  */
4684 static uint64_t stev_ct_maint;
4685 static uint64_t stev_ct_hwerr;
4686 static uint64_t stev_ct_service;
4687 static uint64_t stev_ct_global;
4688 static uint64_t stev_ct_noprefs;
4689 static uint64_t stev_ct_from_uninit;
4690 static uint64_t stev_ct_bad_state;
4691 static uint64_t stev_ct_ovr_prefs;
4692 
4693 static void
4694 dgraph_state_transition_notify(graph_vertex_t *v,
4695     restarter_instance_state_t old_state, restarter_str_t reason)
4696 {
4697         restarter_instance_state_t new_state = v->gv_state;
4698         int stn_transition, maint;
4699         int from, to;
4700         nvlist_t *attr;
4701         fmev_pri_t pri = FMEV_LOPRI;
4702         int raise = 0;
4703 
4704         if ((from = stn_restarter_state(old_state)) == -1 ||
4705             (to = stn_restarter_state(new_state)) == -1) {
4706                 stev_ct_bad_state++;
4707                 return;
4708         }
4709 
4710         stn_transition = from << 16 | to;
4711 
4712         maint = (to == SCF_STATE_MAINT || from == SCF_STATE_MAINT);
4713 
4714         if (maint) {
4715                 /*
4716                  * All transitions to/from maintenance state must raise
4717                  * an event.
4718                  */
4719                 raise++;
4720                 pri = FMEV_HIPRI;
4721                 stev_ct_maint++;
4722         } else if (reason == restarter_str_ct_ev_hwerr) {
4723                 /*
4724                  * All transitions caused by hardware fault must raise
4725                  * an event
4726                  */
4727                 raise++;
4728                 pri = FMEV_HIPRI;
4729                 stev_ct_hwerr++;
4730         } else if (stn_transition & v->gv_stn_tset) {
4731                 /*
4732                  * Specifically enabled event.
4733                  */
4734                 raise++;
4735                 stev_ct_service++;
4736         } else if (from == SCF_STATE_UNINIT) {
4737                 /*
4738                  * Only raise these if specifically selected above.
4739                  */
4740                 stev_ct_from_uninit++;
4741         } else if (stn_transition & stn_global &&
4742             (IS_ENABLED(v) == 1 || to == SCF_STATE_DISABLED)) {
4743                 raise++;
4744                 stev_ct_global++;
4745         } else {
4746                 stev_ct_noprefs++;
4747         }
4748 
4749         if (info_events_all) {
4750                 stev_ct_ovr_prefs++;
4751                 raise++;
4752         }
4753         if (!raise)
4754                 return;
4755 
4756         if (nvlist_alloc(&attr, NV_UNIQUE_NAME, 0) != 0 ||
4757             nvlist_add_string(attr, "fmri", v->gv_name) != 0 ||
4758             nvlist_add_uint32(attr, "reason-version",
4759             restarter_str_version()) || nvlist_add_string(attr, "reason-short",
4760             restarter_get_str_short(reason)) != 0 ||
4761             nvlist_add_string(attr, "reason-long",
4762             restarter_get_str_long(reason)) != 0 ||
4763             nvlist_add_int32(attr, "transition", stn_transition) != 0) {
4764                 log_framework(LOG_WARNING,
4765                     "FMEV: %s could not create nvlist for transition "
4766                     "event: %s\n", v->gv_name, strerror(errno));
4767                 nvlist_free(attr);
4768                 return;
4769         }
4770 
4771         if (fmev_rspublish_nvl(FMEV_RULESET_SMF, "state-transition",
4772             instance_state_str[new_state], pri, attr) != FMEV_SUCCESS) {
4773                 log_framework(LOG_DEBUG,
4774                     "FMEV: %s failed to publish transition event: %s\n",
4775                     v->gv_name, fmev_strerror(fmev_errno));
4776                 nvlist_free(attr);
4777         }
4778 }
4779 
4780 /*
4781  * Find the vertex for inst_name.  If it doesn't exist, return ENOENT.
4782  * Otherwise set its state to state.  If the instance has entered a state
4783  * which requires automatic action, take it (Uninitialized: do
4784  * dgraph_refresh_instance() without the snapshot update.  Disabled: if the
4785  * instance should be enabled, send _ENABLE.  Offline: if the instance should
4786  * be disabled, send _DISABLE, and if its dependencies are satisfied, send
4787  * _START.  Online, Degraded: if the instance wasn't running, update its start
4788  * snapshot.  Maintenance: no action.)
4789  *
4790  * Also fails with ECONNABORTED, or EINVAL if state is invalid.
4791  */
4792 static int
4793 dgraph_set_instance_state(scf_handle_t *h, const char *inst_name,
4794     protocol_states_t *states)
4795 {
4796         graph_vertex_t *v;
4797         int err = 0;
4798         restarter_instance_state_t old_state;
4799         restarter_instance_state_t state = states->ps_state;
4800         restarter_error_t serr = states->ps_err;
4801 
4802         MUTEX_LOCK(&dgraph_lock);
4803 
4804         v = vertex_get_by_name(inst_name);
4805         if (v == NULL) {
4806                 MUTEX_UNLOCK(&dgraph_lock);
4807                 return (ENOENT);
4808         }
4809 
4810         assert(v->gv_type == GVT_INST);
4811 
4812         switch (state) {
4813         case RESTARTER_STATE_UNINIT:
4814         case RESTARTER_STATE_DISABLED:
4815         case RESTARTER_STATE_OFFLINE:
4816         case RESTARTER_STATE_ONLINE:
4817         case RESTARTER_STATE_DEGRADED:
4818         case RESTARTER_STATE_MAINT:
4819                 break;
4820 
4821         default:
4822                 MUTEX_UNLOCK(&dgraph_lock);
4823                 return (EINVAL);
4824         }
4825 
4826         log_framework(LOG_DEBUG, "Graph noting %s %s -> %s.\n", v->gv_name,
4827             instance_state_str[v->gv_state], instance_state_str[state]);
4828 
4829         old_state = v->gv_state;
4830         v->gv_state = state;
4831 
4832         v->gv_reason = states->ps_reason;
4833         err = gt_transition(h, v, serr, old_state);
4834         if (err == 0 && v->gv_state != old_state) {
4835                 dgraph_state_transition_notify(v, old_state, states->ps_reason);
4836         }
4837 
4838         MUTEX_UNLOCK(&dgraph_lock);
4839         return (err);
4840 }
4841 
4842 /*
4843  * Handle state changes during milestone shutdown.  See
4844  * dgraph_set_milestone().  If the repository connection is broken,
4845  * ECONNABORTED will be returned, though a _DISABLE command will be sent for
4846  * the vertex anyway.
4847  */
4848 int
4849 vertex_subgraph_dependencies_shutdown(scf_handle_t *h, graph_vertex_t *v,
4850     restarter_instance_state_t old_state)
4851 {
4852         int was_up, now_up;
4853         int ret = 0;
4854 
4855         assert(v->gv_type == GVT_INST);
4856 
4857         /* Don't care if we're not going to a milestone. */
4858         if (milestone == NULL)
4859                 return (0);
4860 
4861         /* Don't care if we already finished coming down. */
4862         if (non_subgraph_svcs == 0)
4863                 return (0);
4864 
4865         /* Don't care if the service is in the subgraph. */
4866         if (v->gv_flags & GV_INSUBGRAPH)
4867                 return (0);
4868 
4869         /*
4870          * Update non_subgraph_svcs.  It is the number of non-subgraph
4871          * services which are in online, degraded, or offline.
4872          */
4873 
4874         was_up = up_state(old_state);
4875         now_up = up_state(v->gv_state);
4876 
4877         if (!was_up && now_up) {
4878                 ++non_subgraph_svcs;
4879         } else if (was_up && !now_up) {
4880                 --non_subgraph_svcs;
4881 
4882                 if (non_subgraph_svcs == 0) {
4883                         if (halting != -1) {
4884                                 do_uadmin();
4885                         } else if (go_single_user_mode || go_to_level1) {
4886                                 (void) startd_thread_create(single_user_thread,
4887                                     NULL);
4888                         }
4889                         return (0);
4890                 }
4891         }
4892 
4893         /* If this service is a leaf, it should be disabled. */
4894         if ((v->gv_flags & GV_ENABLED) && is_nonsubgraph_leaf(v)) {
4895                 int r;
4896 
4897                 r = disable_service_temporarily(v, h);
4898                 switch (r) {
4899                 case 0:
4900                         break;
4901 
4902                 case ECONNABORTED:
4903                         ret = ECONNABORTED;
4904                         break;
4905 
4906                 default:
4907                         bad_error("disable_service_temporarily", r);
4908                 }
4909         }
4910 
4911         /*
4912          * If the service just came down, propagate the disable to the newly
4913          * exposed leaves.
4914          */
4915         if (was_up && !now_up)
4916                 graph_walk_dependencies(v, disable_nonsubgraph_leaves,
4917                     (void *)h);
4918 
4919         return (ret);
4920 }
4921 
4922 /*
4923  * Decide whether to start up an sulogin thread after a service is
4924  * finished changing state.  Only need to do the full can_come_up()
4925  * evaluation if an instance is changing state, we're not halfway through
4926  * loading the thread, and we aren't shutting down or going to the single
4927  * user milestone.
4928  */
4929 void
4930 graph_transition_sulogin(restarter_instance_state_t state,
4931     restarter_instance_state_t old_state)
4932 {
4933         assert(MUTEX_HELD(&dgraph_lock));
4934 
4935         if (state != old_state && st->st_load_complete &&
4936             !go_single_user_mode && !go_to_level1 &&
4937             halting == -1) {
4938                 if (!sulogin_thread_running && !can_come_up()) {
4939                         (void) startd_thread_create(sulogin_thread, NULL);
4940                         sulogin_thread_running = B_TRUE;
4941                 }
4942         }
4943 }
4944 
4945 /*
4946  * Propagate a start, stop event, or a satisfiability event.
4947  *
4948  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4949  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4950  * full dependent graph to check for newly satisfied nodes.  This is
4951  * necessary for cases when non-direct dependents may be effected but direct
4952  * dependents may not (e.g. for optional_all evaluations, see the
4953  * propagate_satbility() comments).
4954  *
4955  * PROPAGATE_SAT should be used whenever a non-running service moves into
4956  * a state which can satisfy optional dependencies, like disabled or
4957  * maintenance.
4958  */
4959 void
4960 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4961     restarter_error_t rerr)
4962 {
4963         if (type == PROPAGATE_STOP) {
4964                 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4965         } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4966                 graph_walk_dependents(v, propagate_start, NULL);
4967 
4968                 if (type == PROPAGATE_SAT)
4969                         propagate_satbility(v);
4970         } else {
4971 #ifndef NDEBUG
4972                 uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4973                     __LINE__, type);
4974 #endif
4975                 abort();
4976         }
4977 }
4978 
4979 /*
4980  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4981  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4982  * all property group dependencies, and the dependency on the restarter,
4983  * disposing of vertices as appropriate.  If other vertices depend on this
4984  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4985  * returns 0.
4986  */
4987 static int
4988 dgraph_remove_instance(const char *fmri, scf_handle_t *h)
4989 {
4990         graph_vertex_t *v;
4991         graph_edge_t *e;
4992         uu_list_t *old_deps;
4993         int err;
4994 
4995         log_framework(LOG_DEBUG, "Graph engine: Removing %s.\n", fmri);
4996 
4997         MUTEX_LOCK(&dgraph_lock);
4998 
4999         v = vertex_get_by_name(fmri);
5000         if (v == NULL) {
5001                 MUTEX_UNLOCK(&dgraph_lock);
5002                 return (0);
5003         }
5004 
5005         /* Send restarter delete event. */
5006         if (v->gv_flags & GV_CONFIGURED)
5007                 graph_unset_restarter(v);
5008 
5009         if (milestone > MILESTONE_NONE) {
5010                 /*
5011                  * Make a list of v's current dependencies so we can
5012                  * reevaluate their GV_INSUBGRAPH flags after the dependencies
5013                  * are removed.
5014                  */
5015                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
5016 
5017                 err = uu_list_walk(v->gv_dependencies,
5018                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
5019                 assert(err == 0);
5020         }
5021 
5022         delete_instance_dependencies(v, B_TRUE);
5023 
5024         /*
5025          * Deleting an instance can both satisfy and unsatisfy dependencies,
5026          * depending on their type.  First propagate the stop as a RERR_RESTART
5027          * event -- deletion isn't a fault, just a normal stop.  This gives
5028          * dependent services the chance to do a clean shutdown.  Then, mark
5029          * the service as unconfigured and propagate the start event for the
5030          * optional_all dependencies that might have become satisfied.
5031          */
5032         graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
5033 
5034         v->gv_flags &= ~GV_CONFIGURED;
5035         v->gv_flags &= ~GV_DEATHROW;
5036 
5037         graph_walk_dependents(v, propagate_start, NULL);
5038         propagate_satbility(v);
5039 
5040         /*
5041          * If there are no (non-service) dependents, the vertex can be
5042          * completely removed.
5043          */
5044         if (v != milestone && v->gv_refs == 0 &&
5045             uu_list_numnodes(v->gv_dependents) == 1)
5046                 remove_inst_vertex(v);
5047 
5048         if (milestone > MILESTONE_NONE) {
5049                 void *cookie = NULL;
5050 
5051                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5052                         v = e->ge_vertex;
5053 
5054                         if (vertex_unref(v) == VERTEX_INUSE)
5055                                 while (eval_subgraph(v, h) == ECONNABORTED)
5056                                         libscf_handle_rebind(h);
5057 
5058                         startd_free(e, sizeof (*e));
5059                 }
5060 
5061                 uu_list_destroy(old_deps);
5062         }
5063 
5064         MUTEX_UNLOCK(&dgraph_lock);
5065 
5066         return (0);
5067 }
5068 
5069 /*
5070  * Return the eventual (maybe current) milestone in the form of a
5071  * legacy runlevel.
5072  */
5073 static char
5074 target_milestone_as_runlevel()
5075 {
5076         assert(MUTEX_HELD(&dgraph_lock));
5077 
5078         if (milestone == NULL)
5079                 return ('3');
5080         else if (milestone == MILESTONE_NONE)
5081                 return ('0');
5082 
5083         if (strcmp(milestone->gv_name, multi_user_fmri) == 0)
5084                 return ('2');
5085         else if (strcmp(milestone->gv_name, single_user_fmri) == 0)
5086                 return ('S');
5087         else if (strcmp(milestone->gv_name, multi_user_svr_fmri) == 0)
5088                 return ('3');
5089 
5090 #ifndef NDEBUG
5091         (void) fprintf(stderr, "%s:%d: Unknown milestone name \"%s\".\n",
5092             __FILE__, __LINE__, milestone->gv_name);
5093 #endif
5094         abort();
5095         /* NOTREACHED */
5096 }
5097 
5098 static struct {
5099         char    rl;
5100         int     sig;
5101 } init_sigs[] = {
5102         { 'S', SIGBUS },
5103         { '0', SIGINT },
5104         { '1', SIGQUIT },
5105         { '2', SIGILL },
5106         { '3', SIGTRAP },
5107         { '4', SIGIOT },
5108         { '5', SIGEMT },
5109         { '6', SIGFPE },
5110         { 0, 0 }
5111 };
5112 
5113 static void
5114 signal_init(char rl)
5115 {
5116         pid_t init_pid;
5117         int i;
5118 
5119         assert(MUTEX_HELD(&dgraph_lock));
5120 
5121         if (zone_getattr(getzoneid(), ZONE_ATTR_INITPID, &init_pid,
5122             sizeof (init_pid)) != sizeof (init_pid)) {
5123                 log_error(LOG_NOTICE, "Could not get pid to signal init.\n");
5124                 return;
5125         }
5126 
5127         for (i = 0; init_sigs[i].rl != 0; ++i)
5128                 if (init_sigs[i].rl == rl)
5129                         break;
5130 
5131         if (init_sigs[i].rl != 0) {
5132                 if (kill(init_pid, init_sigs[i].sig) != 0) {
5133                         switch (errno) {
5134                         case EPERM:
5135                         case ESRCH:
5136                                 log_error(LOG_NOTICE, "Could not signal init: "
5137                                     "%s.\n", strerror(errno));
5138                                 break;
5139 
5140                         case EINVAL:
5141                         default:
5142                                 bad_error("kill", errno);
5143                         }
5144                 }
5145         }
5146 }
5147 
5148 /*
5149  * This is called when one of the major milestones changes state, or when
5150  * init is signalled and tells us it was told to change runlevel.  We wait
5151  * to reach the milestone because this allows /etc/inittab entries to retain
5152  * some boot ordering: historically, entries could place themselves before/after
5153  * the running of /sbin/rcX scripts but we can no longer make the
5154  * distinction because the /sbin/rcX scripts no longer exist as punctuation
5155  * marks in /etc/inittab.
5156  *
5157  * Also, we only trigger an update when we reach the eventual target
5158  * milestone: without this, an /etc/inittab entry marked only for
5159  * runlevel 2 would be executed for runlevel 3, which is not how
5160  * /etc/inittab entries work.
5161  *
5162  * If we're single user coming online, then we set utmpx to the target
5163  * runlevel so that legacy scripts can work as expected.
5164  */
5165 static void
5166 graph_runlevel_changed(char rl, int online)
5167 {
5168         char trl;
5169 
5170         assert(MUTEX_HELD(&dgraph_lock));
5171 
5172         trl = target_milestone_as_runlevel();
5173 
5174         if (online) {
5175                 if (rl == trl) {
5176                         current_runlevel = trl;
5177                         signal_init(trl);
5178                 } else if (rl == 'S') {
5179                         /*
5180                          * At boot, set the entry early for the benefit of the
5181                          * legacy init scripts.
5182                          */
5183                         utmpx_set_runlevel(trl, 'S', B_FALSE);
5184                 }
5185         } else {
5186                 if (rl == '3' && trl == '2') {
5187                         current_runlevel = trl;
5188                         signal_init(trl);
5189                 } else if (rl == '2' && trl == 'S') {
5190                         current_runlevel = trl;
5191                         signal_init(trl);
5192                 }
5193         }
5194 }
5195 
5196 /*
5197  * Move to a backwards-compatible runlevel by executing the appropriate
5198  * /etc/rc?.d/K* scripts and/or setting the milestone.
5199  *
5200  * Returns
5201  *   0 - success
5202  *   ECONNRESET - success, but handle was reset
5203  *   ECONNABORTED - repository connection broken
5204  *   ECANCELED - pg was deleted
5205  */
5206 static int
5207 dgraph_set_runlevel(scf_propertygroup_t *pg, scf_property_t *prop)
5208 {
5209         char rl;
5210         scf_handle_t *h;
5211         int r;
5212         const char *ms = NULL;  /* what to commit as options/milestone */
5213         boolean_t rebound = B_FALSE;
5214         int mark_rl = 0;
5215 
5216         const char * const stop = "stop";
5217 
5218         r = libscf_extract_runlevel(prop, &rl);
5219         switch (r) {
5220         case 0:
5221                 break;
5222 
5223         case ECONNABORTED:
5224         case ECANCELED:
5225                 return (r);
5226 
5227         case EINVAL:
5228         case ENOENT:
5229                 log_error(LOG_WARNING, "runlevel property is misconfigured; "
5230                     "ignoring.\n");
5231                 /* delete the bad property */
5232                 goto nolock_out;
5233 
5234         default:
5235                 bad_error("libscf_extract_runlevel", r);
5236         }
5237 
5238         switch (rl) {
5239         case 's':
5240                 rl = 'S';
5241                 /* FALLTHROUGH */
5242 
5243         case 'S':
5244         case '2':
5245         case '3':
5246                 /*
5247                  * These cases cause a milestone change, so
5248                  * graph_runlevel_changed() will eventually deal with
5249                  * signalling init.
5250                  */
5251                 break;
5252 
5253         case '0':
5254         case '1':
5255         case '4':
5256         case '5':
5257         case '6':
5258                 mark_rl = 1;
5259                 break;
5260 
5261         default:
5262                 log_framework(LOG_NOTICE, "Unknown runlevel '%c'.\n", rl);
5263                 ms = NULL;
5264                 goto nolock_out;
5265         }
5266 
5267         h = scf_pg_handle(pg);
5268 
5269         MUTEX_LOCK(&dgraph_lock);
5270 
5271         /*
5272          * Since this triggers no milestone changes, force it by hand.
5273          */
5274         if (current_runlevel == '4' && rl == '3')
5275                 mark_rl = 1;
5276 
5277         /*
5278          * 1. If we are here after an "init X":
5279          *
5280          * init X
5281          *      init/lscf_set_runlevel()
5282          *              process_pg_event()
5283          *              dgraph_set_runlevel()
5284          *
5285          * then we haven't passed through graph_runlevel_changed() yet,
5286          * therefore 'current_runlevel' has not changed for sure but 'rl' has.
5287          * In consequence, if 'rl' is lower than 'current_runlevel', we change
5288          * the system runlevel and execute the appropriate /etc/rc?.d/K* scripts
5289          * past this test.
5290          *
5291          * 2. On the other hand, if we are here after a "svcadm milestone":
5292          *
5293          * svcadm milestone X
5294          *      dgraph_set_milestone()
5295          *              handle_graph_update_event()
5296          *              dgraph_set_instance_state()
5297          *              graph_post_X_[online|offline]()
5298          *              graph_runlevel_changed()
5299          *              signal_init()
5300          *                      init/lscf_set_runlevel()
5301          *                              process_pg_event()
5302          *                              dgraph_set_runlevel()
5303          *
5304          * then we already passed through graph_runlevel_changed() (by the way
5305          * of dgraph_set_milestone()) and 'current_runlevel' may have changed
5306          * and already be equal to 'rl' so we are going to return immediately
5307          * from dgraph_set_runlevel() without changing the system runlevel and
5308          * without executing the /etc/rc?.d/K* scripts.
5309          */
5310         if (rl == current_runlevel) {
5311                 ms = NULL;
5312                 goto out;
5313         }
5314 
5315         log_framework(LOG_DEBUG, "Changing to runlevel '%c'.\n", rl);
5316 
5317         /*
5318          * Make sure stop rc scripts see the new settings via who -r.
5319          */
5320         utmpx_set_runlevel(rl, current_runlevel, B_TRUE);
5321 
5322         /*
5323          * Some run levels don't have a direct correspondence to any
5324          * milestones, so we have to signal init directly.
5325          */
5326         if (mark_rl) {
5327                 current_runlevel = rl;
5328                 signal_init(rl);
5329         }
5330 
5331         switch (rl) {
5332         case 'S':
5333                 uu_warn("The system is coming down for administration.  "
5334                     "Please wait.\n");
5335                 fork_rc_script(rl, stop, B_FALSE);
5336                 ms = single_user_fmri;
5337                 go_single_user_mode = B_TRUE;
5338                 break;
5339 
5340         case '0':
5341                 halting_time = time(NULL);
5342                 fork_rc_script(rl, stop, B_TRUE);
5343                 halting = AD_HALT;
5344                 goto uadmin;
5345 
5346         case '5':
5347                 halting_time = time(NULL);
5348                 fork_rc_script(rl, stop, B_TRUE);
5349                 halting = AD_POWEROFF;
5350                 goto uadmin;
5351 
5352         case '6':
5353                 halting_time = time(NULL);
5354                 fork_rc_script(rl, stop, B_TRUE);
5355                 if (scf_is_fastboot_default() && getzoneid() == GLOBAL_ZONEID)
5356                         halting = AD_FASTREBOOT;
5357                 else
5358                         halting = AD_BOOT;
5359 
5360 uadmin:
5361                 uu_warn("The system is coming down.  Please wait.\n");
5362                 ms = "none";
5363 
5364                 /*
5365                  * We can't wait until all services are offline since this
5366                  * thread is responsible for taking them offline.  Instead we
5367                  * set halting to the second argument for uadmin() and call
5368                  * do_uadmin() from dgraph_set_instance_state() when
5369                  * appropriate.
5370                  */
5371                 break;
5372 
5373         case '1':
5374                 if (current_runlevel != 'S') {
5375                         uu_warn("Changing to state 1.\n");
5376                         fork_rc_script(rl, stop, B_FALSE);
5377                 } else {
5378                         uu_warn("The system is coming up for administration.  "
5379                             "Please wait.\n");
5380                 }
5381                 ms = single_user_fmri;
5382                 go_to_level1 = B_TRUE;
5383                 break;
5384 
5385         case '2':
5386                 if (current_runlevel == '3' || current_runlevel == '4')
5387                         fork_rc_script(rl, stop, B_FALSE);
5388                 ms = multi_user_fmri;
5389                 break;
5390 
5391         case '3':
5392         case '4':
5393                 ms = "all";
5394                 break;
5395 
5396         default:
5397 #ifndef NDEBUG
5398                 (void) fprintf(stderr, "%s:%d: Uncaught case %d ('%c').\n",
5399                     __FILE__, __LINE__, rl, rl);
5400 #endif
5401                 abort();
5402         }
5403 
5404 out:
5405         MUTEX_UNLOCK(&dgraph_lock);
5406 
5407 nolock_out:
5408         switch (r = libscf_clear_runlevel(pg, ms)) {
5409         case 0:
5410                 break;
5411 
5412         case ECONNABORTED:
5413                 libscf_handle_rebind(h);
5414                 rebound = B_TRUE;
5415                 goto nolock_out;
5416 
5417         case ECANCELED:
5418                 break;
5419 
5420         case EPERM:
5421         case EACCES:
5422         case EROFS:
5423                 log_error(LOG_NOTICE, "Could not delete \"%s/%s\" property: "
5424                     "%s.\n", SCF_PG_OPTIONS, "runlevel", strerror(r));
5425                 break;
5426 
5427         default:
5428                 bad_error("libscf_clear_runlevel", r);
5429         }
5430 
5431         return (rebound ? ECONNRESET : 0);
5432 }
5433 
5434 /*
5435  * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5436  * to the instances that are supposed to go offline during an
5437  * administrative disable operation.
5438  */
5439 static int
5440 mark_subtree(graph_edge_t *e, void *arg)
5441 {
5442         graph_vertex_t *v;
5443         int r;
5444 
5445         v = e->ge_vertex;
5446 
5447         /* If it's already in the subgraph, skip. */
5448         if (v->gv_flags & GV_TOOFFLINE)
5449                 return (UU_WALK_NEXT);
5450 
5451         switch (v->gv_type) {
5452         case GVT_INST:
5453                 /* If the instance is already disabled, skip it. */
5454                 if (!(v->gv_flags & GV_ENABLED))
5455                         return (UU_WALK_NEXT);
5456 
5457                 v->gv_flags |= GV_TOOFFLINE;
5458                 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5459                 break;
5460         case GVT_GROUP:
5461                 /*
5462                  * Skip all excluded dependencies and decide whether to offline
5463                  * the service based on the restart_on attribute.
5464                  */
5465                 if (is_depgrp_bypassed(v))
5466                         return (UU_WALK_NEXT);
5467                 break;
5468         }
5469 
5470         r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5471             0);
5472         assert(r == 0);
5473         return (UU_WALK_NEXT);
5474 }
5475 
5476 static int
5477 mark_subgraph(graph_edge_t *e, void *arg)
5478 {
5479         graph_vertex_t *v;
5480         int r;
5481         int optional = (int)arg;
5482 
5483         v = e->ge_vertex;
5484 
5485         /* If it's already in the subgraph, skip. */
5486         if (v->gv_flags & GV_INSUBGRAPH)
5487                 return (UU_WALK_NEXT);
5488 
5489         /*
5490          * Keep track if walk has entered an optional dependency group
5491          */
5492         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_OPTIONAL_ALL) {
5493                 optional = 1;
5494         }
5495         /*
5496          * Quit if we are in an optional dependency group and the instance
5497          * is disabled
5498          */
5499         if (optional && (v->gv_type == GVT_INST) &&
5500             (!(v->gv_flags & GV_ENBLD_NOOVR)))
5501                 return (UU_WALK_NEXT);
5502 
5503         v->gv_flags |= GV_INSUBGRAPH;
5504 
5505         /* Skip all excluded dependencies. */
5506         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
5507                 return (UU_WALK_NEXT);
5508 
5509         r = uu_list_walk(v->gv_dependencies, (uu_walk_fn_t *)mark_subgraph,
5510             (void *)optional, 0);
5511         assert(r == 0);
5512         return (UU_WALK_NEXT);
5513 }
5514 
5515 /*
5516  * Bring down all services which are not dependencies of fmri.  The
5517  * dependencies of fmri (direct & indirect) will constitute the "subgraph",
5518  * and will have the GV_INSUBGRAPH flag set.  The rest must be brought down,
5519  * which means the state is "disabled", "maintenance", or "uninitialized".  We
5520  * could consider "offline" to be down, and refrain from sending start
5521  * commands for such services, but that's not strictly necessary, so we'll
5522  * decline to intrude on the state machine.  It would probably confuse users
5523  * anyway.
5524  *
5525  * The services should be brought down in reverse-dependency order, so we
5526  * can't do it all at once here.  We initiate by override-disabling the leaves
5527  * of the dependency tree -- those services which are up but have no
5528  * dependents which are up.  When they come down,
5529  * vertex_subgraph_dependencies_shutdown() will override-disable the newly
5530  * exposed leaves.  Perseverance will ensure completion.
5531  *
5532  * Sometimes we need to take action when the transition is complete, like
5533  * start sulogin or halt the system.  To tell when we're done, we initialize
5534  * non_subgraph_svcs here to be the number of services which need to come
5535  * down.  As each does, we decrement the counter.  When it hits zero, we take
5536  * the appropriate action.  See vertex_subgraph_dependencies_shutdown().
5537  *
5538  * In case we're coming up, we also remove any enable-overrides for the
5539  * services which are dependencies of fmri.
5540  *
5541  * If norepository is true, the function will not change the repository.
5542  *
5543  * The decision to change the system run level in accordance with the milestone
5544  * is taken in dgraph_set_runlevel().
5545  *
5546  * Returns
5547  *   0 - success
5548  *   ECONNRESET - success, but handle was rebound
5549  *   EINVAL - fmri is invalid (error is logged)
5550  *   EALREADY - the milestone is already set to fmri
5551  *   ENOENT - a configured vertex does not exist for fmri (an error is logged)
5552  */
5553 static int
5554 dgraph_set_milestone(const char *fmri, scf_handle_t *h, boolean_t norepository)
5555 {
5556         const char *cfmri, *fs;
5557         graph_vertex_t *nm, *v;
5558         int ret = 0, r;
5559         scf_instance_t *inst;
5560         boolean_t isall, isnone, rebound = B_FALSE;
5561 
5562         /* Validate fmri */
5563         isall = (strcmp(fmri, "all") == 0);
5564         isnone = (strcmp(fmri, "none") == 0);
5565 
5566         if (!isall && !isnone) {
5567                 if (fmri_canonify(fmri, (char **)&cfmri, B_FALSE) == EINVAL)
5568                         goto reject;
5569 
5570                 if (strcmp(cfmri, single_user_fmri) != 0 &&
5571                     strcmp(cfmri, multi_user_fmri) != 0 &&
5572                     strcmp(cfmri, multi_user_svr_fmri) != 0) {
5573                         startd_free((void *)cfmri, max_scf_fmri_size);
5574 reject:
5575                         log_framework(LOG_WARNING,
5576                             "Rejecting request for invalid milestone \"%s\".\n",
5577                             fmri);
5578                         return (EINVAL);
5579                 }
5580         }
5581 
5582         inst = safe_scf_instance_create(h);
5583 
5584         MUTEX_LOCK(&dgraph_lock);
5585 
5586         if (milestone == NULL) {
5587                 if (isall) {
5588                         log_framework(LOG_DEBUG,
5589                             "Milestone already set to all.\n");
5590                         ret = EALREADY;
5591                         goto out;
5592                 }
5593         } else if (milestone == MILESTONE_NONE) {
5594                 if (isnone) {
5595                         log_framework(LOG_DEBUG,
5596                             "Milestone already set to none.\n");
5597                         ret = EALREADY;
5598                         goto out;
5599                 }
5600         } else {
5601                 if (!isall && !isnone &&
5602                     strcmp(cfmri, milestone->gv_name) == 0) {
5603                         log_framework(LOG_DEBUG,
5604                             "Milestone already set to %s.\n", cfmri);
5605                         ret = EALREADY;
5606                         goto out;
5607                 }
5608         }
5609 
5610         if (!isall && !isnone) {
5611                 nm = vertex_get_by_name(cfmri);
5612                 if (nm == NULL || !(nm->gv_flags & GV_CONFIGURED)) {
5613                         log_framework(LOG_WARNING, "Cannot set milestone to %s "
5614                             "because no such service exists.\n", cfmri);
5615                         ret = ENOENT;
5616                         goto out;
5617                 }
5618         }
5619 
5620         log_framework(LOG_DEBUG, "Changing milestone to %s.\n", fmri);
5621 
5622         /*
5623          * Set milestone, removing the old one if this was the last reference.
5624          */
5625         if (milestone > MILESTONE_NONE)
5626                 (void) vertex_unref(milestone);
5627 
5628         if (isall)
5629                 milestone = NULL;
5630         else if (isnone)
5631                 milestone = MILESTONE_NONE;
5632         else {
5633                 milestone = nm;
5634                 /* milestone should count as a reference */
5635                 vertex_ref(milestone);
5636         }
5637 
5638         /* Clear all GV_INSUBGRAPH bits. */
5639         for (v = uu_list_first(dgraph); v != NULL; v = uu_list_next(dgraph, v))
5640                 v->gv_flags &= ~GV_INSUBGRAPH;
5641 
5642         if (!isall && !isnone) {
5643                 /* Set GV_INSUBGRAPH for milestone & descendents. */
5644                 milestone->gv_flags |= GV_INSUBGRAPH;
5645 
5646                 r = uu_list_walk(milestone->gv_dependencies,
5647                     (uu_walk_fn_t *)mark_subgraph, NULL, 0);
5648                 assert(r == 0);
5649         }
5650 
5651         /* Un-override services in the subgraph & override-disable the rest. */
5652         if (norepository)
5653                 goto out;
5654 
5655         non_subgraph_svcs = 0;
5656         for (v = uu_list_first(dgraph);
5657             v != NULL;
5658             v = uu_list_next(dgraph, v)) {
5659                 if (v->gv_type != GVT_INST ||
5660                     (v->gv_flags & GV_CONFIGURED) == 0)
5661                         continue;
5662 
5663 again:
5664                 r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
5665                     NULL, NULL, SCF_DECODE_FMRI_EXACT);
5666                 if (r != 0) {
5667                         switch (scf_error()) {
5668                         case SCF_ERROR_CONNECTION_BROKEN:
5669                         default:
5670                                 libscf_handle_rebind(h);
5671                                 rebound = B_TRUE;
5672                                 goto again;
5673 
5674                         case SCF_ERROR_NOT_FOUND:
5675                                 continue;
5676 
5677                         case SCF_ERROR_HANDLE_MISMATCH:
5678                         case SCF_ERROR_INVALID_ARGUMENT:
5679                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5680                         case SCF_ERROR_NOT_BOUND:
5681                                 bad_error("scf_handle_decode_fmri",
5682                                     scf_error());
5683                         }
5684                 }
5685 
5686                 if (isall || (v->gv_flags & GV_INSUBGRAPH)) {
5687                         r = libscf_delete_enable_ovr(inst);
5688                         fs = "libscf_delete_enable_ovr";
5689                 } else {
5690                         assert(isnone || (v->gv_flags & GV_INSUBGRAPH) == 0);
5691 
5692                         /*
5693                          * Services which are up need to come down before
5694                          * we're done, but we can only disable the leaves
5695                          * here.
5696                          */
5697 
5698                         if (up_state(v->gv_state))
5699                                 ++non_subgraph_svcs;
5700 
5701                         /* If it's already disabled, don't bother. */
5702                         if ((v->gv_flags & GV_ENABLED) == 0)
5703                                 continue;
5704 
5705                         if (!is_nonsubgraph_leaf(v))
5706                                 continue;
5707 
5708                         r = libscf_set_enable_ovr(inst, 0);
5709                         fs = "libscf_set_enable_ovr";
5710                 }
5711                 switch (r) {
5712                 case 0:
5713                 case ECANCELED:
5714                         break;
5715 
5716                 case ECONNABORTED:
5717                         libscf_handle_rebind(h);
5718                         rebound = B_TRUE;
5719                         goto again;
5720 
5721                 case EPERM:
5722                 case EROFS:
5723                         log_error(LOG_WARNING,
5724                             "Could not set %s/%s for %s: %s.\n",
5725                             SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
5726                             v->gv_name, strerror(r));
5727                         break;
5728 
5729                 default:
5730                         bad_error(fs, r);
5731                 }
5732         }
5733 
5734         if (halting != -1) {
5735                 if (non_subgraph_svcs > 1)
5736                         uu_warn("%d system services are now being stopped.\n",
5737                             non_subgraph_svcs);
5738                 else if (non_subgraph_svcs == 1)
5739                         uu_warn("One system service is now being stopped.\n");
5740                 else if (non_subgraph_svcs == 0)
5741                         do_uadmin();
5742         }
5743 
5744         ret = rebound ? ECONNRESET : 0;
5745 
5746 out:
5747         MUTEX_UNLOCK(&dgraph_lock);
5748         if (!isall && !isnone)
5749                 startd_free((void *)cfmri, max_scf_fmri_size);
5750         scf_instance_destroy(inst);
5751         return (ret);
5752 }
5753 
5754 
5755 /*
5756  * Returns 0, ECONNABORTED, or EINVAL.
5757  */
5758 static int
5759 handle_graph_update_event(scf_handle_t *h, graph_protocol_event_t *e)
5760 {
5761         int r;
5762 
5763         switch (e->gpe_type) {
5764         case GRAPH_UPDATE_RELOAD_GRAPH:
5765                 log_error(LOG_WARNING,
5766                     "graph_event: reload graph unimplemented\n");
5767                 break;
5768 
5769         case GRAPH_UPDATE_STATE_CHANGE: {
5770                 protocol_states_t *states = e->gpe_data;
5771 
5772                 switch (r = dgraph_set_instance_state(h, e->gpe_inst, states)) {
5773                 case 0:
5774                 case ENOENT:
5775                         break;
5776 
5777                 case ECONNABORTED:
5778                         return (ECONNABORTED);
5779 
5780                 case EINVAL:
5781                 default:
5782 #ifndef NDEBUG
5783                         (void) fprintf(stderr, "dgraph_set_instance_state() "
5784                             "failed with unexpected error %d at %s:%d.\n", r,
5785                             __FILE__, __LINE__);
5786 #endif
5787                         abort();
5788                 }
5789 
5790                 startd_free(states, sizeof (protocol_states_t));
5791                 break;
5792         }
5793 
5794         default:
5795                 log_error(LOG_WARNING,
5796                     "graph_event_loop received an unknown event: %d\n",
5797                     e->gpe_type);
5798                 break;
5799         }
5800 
5801         return (0);
5802 }
5803 
5804 /*
5805  * graph_event_thread()
5806  *    Wait for state changes from the restarters.
5807  */
5808 /*ARGSUSED*/
5809 void *
5810 graph_event_thread(void *unused)
5811 {
5812         scf_handle_t *h;
5813         int err;
5814 
5815         h = libscf_handle_create_bound_loop();
5816 
5817         /*CONSTCOND*/
5818         while (1) {
5819                 graph_protocol_event_t *e;
5820 
5821                 MUTEX_LOCK(&gu->gu_lock);
5822 
5823                 while (gu->gu_wakeup == 0)
5824                         (void) pthread_cond_wait(&gu->gu_cv, &gu->gu_lock);
5825 
5826                 gu->gu_wakeup = 0;
5827 
5828                 while ((e = graph_event_dequeue()) != NULL) {
5829                         MUTEX_LOCK(&e->gpe_lock);
5830                         MUTEX_UNLOCK(&gu->gu_lock);
5831 
5832                         while ((err = handle_graph_update_event(h, e)) ==
5833                             ECONNABORTED)
5834                                 libscf_handle_rebind(h);
5835 
5836                         if (err == 0)
5837                                 graph_event_release(e);
5838                         else
5839                                 graph_event_requeue(e);
5840 
5841                         MUTEX_LOCK(&gu->gu_lock);
5842                 }
5843 
5844                 MUTEX_UNLOCK(&gu->gu_lock);
5845         }
5846 
5847         /*
5848          * Unreachable for now -- there's currently no graceful cleanup
5849          * called on exit().
5850          */
5851         MUTEX_UNLOCK(&gu->gu_lock);
5852         scf_handle_destroy(h);
5853         return (NULL);
5854 }
5855 
5856 static void
5857 set_initial_milestone(scf_handle_t *h)
5858 {
5859         scf_instance_t *inst;
5860         char *fmri, *cfmri;
5861         size_t sz;
5862         int r;
5863 
5864         inst = safe_scf_instance_create(h);
5865         fmri = startd_alloc(max_scf_fmri_size);
5866 
5867         /*
5868          * If -m milestone= was specified, we want to set options_ovr/milestone
5869          * to it.  Otherwise we want to read what the milestone should be set
5870          * to.  Either way we need our inst.
5871          */
5872 get_self:
5873         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
5874             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5875                 switch (scf_error()) {
5876                 case SCF_ERROR_CONNECTION_BROKEN:
5877                         libscf_handle_rebind(h);
5878                         goto get_self;
5879 
5880                 case SCF_ERROR_NOT_FOUND:
5881                         if (st->st_subgraph != NULL &&
5882                             st->st_subgraph[0] != '\0') {
5883                                 sz = strlcpy(fmri, st->st_subgraph,
5884                                     max_scf_fmri_size);
5885                                 assert(sz < max_scf_fmri_size);
5886                         } else {
5887                                 fmri[0] = '\0';
5888                         }
5889                         break;
5890 
5891                 case SCF_ERROR_INVALID_ARGUMENT:
5892                 case SCF_ERROR_CONSTRAINT_VIOLATED:
5893                 case SCF_ERROR_HANDLE_MISMATCH:
5894                 default:
5895                         bad_error("scf_handle_decode_fmri", scf_error());
5896                 }
5897         } else {
5898                 if (st->st_subgraph != NULL && st->st_subgraph[0] != '\0') {
5899                         scf_propertygroup_t *pg;
5900 
5901                         pg = safe_scf_pg_create(h);
5902 
5903                         sz = strlcpy(fmri, st->st_subgraph, max_scf_fmri_size);
5904                         assert(sz < max_scf_fmri_size);
5905 
5906                         r = libscf_inst_get_or_add_pg(inst, SCF_PG_OPTIONS_OVR,
5907                             SCF_PG_OPTIONS_OVR_TYPE, SCF_PG_OPTIONS_OVR_FLAGS,
5908                             pg);
5909                         switch (r) {
5910                         case 0:
5911                                 break;
5912 
5913                         case ECONNABORTED:
5914                                 libscf_handle_rebind(h);
5915                                 goto get_self;
5916 
5917                         case EPERM:
5918                         case EACCES:
5919                         case EROFS:
5920                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5921                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5922                                     SCF_PROPERTY_MILESTONE, strerror(r));
5923                                 /* FALLTHROUGH */
5924 
5925                         case ECANCELED:
5926                                 sz = strlcpy(fmri, st->st_subgraph,
5927                                     max_scf_fmri_size);
5928                                 assert(sz < max_scf_fmri_size);
5929                                 break;
5930 
5931                         default:
5932                                 bad_error("libscf_inst_get_or_add_pg", r);
5933                         }
5934 
5935                         r = libscf_clear_runlevel(pg, fmri);
5936                         switch (r) {
5937                         case 0:
5938                                 break;
5939 
5940                         case ECONNABORTED:
5941                                 libscf_handle_rebind(h);
5942                                 goto get_self;
5943 
5944                         case EPERM:
5945                         case EACCES:
5946                         case EROFS:
5947                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5948                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5949                                     SCF_PROPERTY_MILESTONE, strerror(r));
5950                                 /* FALLTHROUGH */
5951 
5952                         case ECANCELED:
5953                                 sz = strlcpy(fmri, st->st_subgraph,
5954                                     max_scf_fmri_size);
5955                                 assert(sz < max_scf_fmri_size);
5956                                 break;
5957 
5958                         default:
5959                                 bad_error("libscf_clear_runlevel", r);
5960                         }
5961 
5962                         scf_pg_destroy(pg);
5963                 } else {
5964                         scf_property_t *prop;
5965                         scf_value_t *val;
5966 
5967                         prop = safe_scf_property_create(h);
5968                         val = safe_scf_value_create(h);
5969 
5970                         r = libscf_get_milestone(inst, prop, val, fmri,
5971                             max_scf_fmri_size);
5972                         switch (r) {
5973                         case 0:
5974                                 break;
5975 
5976                         case ECONNABORTED:
5977                                 libscf_handle_rebind(h);
5978                                 goto get_self;
5979 
5980                         case EINVAL:
5981                                 log_error(LOG_WARNING, "Milestone property is "
5982                                     "misconfigured.  Defaulting to \"all\".\n");
5983                                 /* FALLTHROUGH */
5984 
5985                         case ECANCELED:
5986                         case ENOENT:
5987                                 fmri[0] = '\0';
5988                                 break;
5989 
5990                         default:
5991                                 bad_error("libscf_get_milestone", r);
5992                         }
5993 
5994                         scf_value_destroy(val);
5995                         scf_property_destroy(prop);
5996                 }
5997         }
5998 
5999         if (fmri[0] == '\0' || strcmp(fmri, "all") == 0)
6000                 goto out;
6001 
6002         if (strcmp(fmri, "none") != 0) {
6003 retry:
6004                 if (scf_handle_decode_fmri(h, fmri, NULL, NULL, inst, NULL,
6005                     NULL, SCF_DECODE_FMRI_EXACT) != 0) {
6006                         switch (scf_error()) {
6007                         case SCF_ERROR_INVALID_ARGUMENT:
6008                                 log_error(LOG_WARNING,
6009                                     "Requested milestone \"%s\" is invalid.  "
6010                                     "Reverting to \"all\".\n", fmri);
6011                                 goto out;
6012 
6013                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6014                                 log_error(LOG_WARNING, "Requested milestone "
6015                                     "\"%s\" does not specify an instance.  "
6016                                     "Reverting to \"all\".\n", fmri);
6017                                 goto out;
6018 
6019                         case SCF_ERROR_CONNECTION_BROKEN:
6020                                 libscf_handle_rebind(h);
6021                                 goto retry;
6022 
6023                         case SCF_ERROR_NOT_FOUND:
6024                                 log_error(LOG_WARNING, "Requested milestone "
6025                                     "\"%s\" not in repository.  Reverting to "
6026                                     "\"all\".\n", fmri);
6027                                 goto out;
6028 
6029                         case SCF_ERROR_HANDLE_MISMATCH:
6030                         default:
6031                                 bad_error("scf_handle_decode_fmri",
6032                                     scf_error());
6033                         }
6034                 }
6035 
6036                 r = fmri_canonify(fmri, &cfmri, B_FALSE);
6037                 assert(r == 0);
6038 
6039                 r = dgraph_add_instance(cfmri, inst, B_TRUE);
6040                 startd_free(cfmri, max_scf_fmri_size);
6041                 switch (r) {
6042                 case 0:
6043                         break;
6044 
6045                 case ECONNABORTED:
6046                         goto retry;
6047 
6048                 case EINVAL:
6049                         log_error(LOG_WARNING,
6050                             "Requested milestone \"%s\" is invalid.  "
6051                             "Reverting to \"all\".\n", fmri);
6052                         goto out;
6053 
6054                 case ECANCELED:
6055                         log_error(LOG_WARNING,
6056                             "Requested milestone \"%s\" not "
6057                             "in repository.  Reverting to \"all\".\n",
6058                             fmri);
6059                         goto out;
6060 
6061                 case EEXIST:
6062                 default:
6063                         bad_error("dgraph_add_instance", r);
6064                 }
6065         }
6066 
6067         log_console(LOG_INFO, "Booting to milestone \"%s\".\n", fmri);
6068 
6069         r = dgraph_set_milestone(fmri, h, B_FALSE);
6070         switch (r) {
6071         case 0:
6072         case ECONNRESET:
6073         case EALREADY:
6074                 break;
6075 
6076         case EINVAL:
6077         case ENOENT:
6078         default:
6079                 bad_error("dgraph_set_milestone", r);
6080         }
6081 
6082 out:
6083         startd_free(fmri, max_scf_fmri_size);
6084         scf_instance_destroy(inst);
6085 }
6086 
6087 void
6088 set_restart_milestone(scf_handle_t *h)
6089 {
6090         scf_instance_t *inst;
6091         scf_property_t *prop;
6092         scf_value_t *val;
6093         char *fmri;
6094         int r;
6095 
6096         inst = safe_scf_instance_create(h);
6097 
6098 get_self:
6099         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL,
6100             inst, NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
6101                 switch (scf_error()) {
6102                 case SCF_ERROR_CONNECTION_BROKEN:
6103                         libscf_handle_rebind(h);
6104                         goto get_self;
6105 
6106                 case SCF_ERROR_NOT_FOUND:
6107                         break;
6108 
6109                 case SCF_ERROR_INVALID_ARGUMENT:
6110                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6111                 case SCF_ERROR_HANDLE_MISMATCH:
6112                 default:
6113                         bad_error("scf_handle_decode_fmri", scf_error());
6114                 }
6115 
6116                 scf_instance_destroy(inst);
6117                 return;
6118         }
6119 
6120         prop = safe_scf_property_create(h);
6121         val = safe_scf_value_create(h);
6122         fmri = startd_alloc(max_scf_fmri_size);
6123 
6124         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6125         switch (r) {
6126         case 0:
6127                 break;
6128 
6129         case ECONNABORTED:
6130                 libscf_handle_rebind(h);
6131                 goto get_self;
6132 
6133         case ECANCELED:
6134         case ENOENT:
6135         case EINVAL:
6136                 goto out;
6137 
6138         default:
6139                 bad_error("libscf_get_milestone", r);
6140         }
6141 
6142         r = dgraph_set_milestone(fmri, h, B_TRUE);
6143         switch (r) {
6144         case 0:
6145         case ECONNRESET:
6146         case EALREADY:
6147         case EINVAL:
6148         case ENOENT:
6149                 break;
6150 
6151         default:
6152                 bad_error("dgraph_set_milestone", r);
6153         }
6154 
6155 out:
6156         startd_free(fmri, max_scf_fmri_size);
6157         scf_value_destroy(val);
6158         scf_property_destroy(prop);
6159         scf_instance_destroy(inst);
6160 }
6161 
6162 /*
6163  * void *graph_thread(void *)
6164  *
6165  * Graph management thread.
6166  */
6167 /*ARGSUSED*/
6168 void *
6169 graph_thread(void *arg)
6170 {
6171         scf_handle_t *h;
6172         int err;
6173 
6174         h = libscf_handle_create_bound_loop();
6175 
6176         if (st->st_initial)
6177                 set_initial_milestone(h);
6178 
6179         MUTEX_LOCK(&dgraph_lock);
6180         initial_milestone_set = B_TRUE;
6181         err = pthread_cond_broadcast(&initial_milestone_cv);
6182         assert(err == 0);
6183         MUTEX_UNLOCK(&dgraph_lock);
6184 
6185         libscf_populate_graph(h);
6186 
6187         if (!st->st_initial)
6188                 set_restart_milestone(h);
6189 
6190         MUTEX_LOCK(&st->st_load_lock);
6191         st->st_load_complete = 1;
6192         (void) pthread_cond_broadcast(&st->st_load_cv);
6193         MUTEX_UNLOCK(&st->st_load_lock);
6194 
6195         MUTEX_LOCK(&dgraph_lock);
6196         /*
6197          * Now that we've set st_load_complete we need to check can_come_up()
6198          * since if we booted to a milestone, then there won't be any more
6199          * state updates.
6200          */
6201         if (!go_single_user_mode && !go_to_level1 &&
6202             halting == -1) {
6203                 if (!sulogin_thread_running && !can_come_up()) {
6204                         (void) startd_thread_create(sulogin_thread, NULL);
6205                         sulogin_thread_running = B_TRUE;
6206                 }
6207         }
6208         MUTEX_UNLOCK(&dgraph_lock);
6209 
6210         (void) pthread_mutex_lock(&gu->gu_freeze_lock);
6211 
6212         /*CONSTCOND*/
6213         while (1) {
6214                 (void) pthread_cond_wait(&gu->gu_freeze_cv,
6215                     &gu->gu_freeze_lock);
6216         }
6217 
6218         /*
6219          * Unreachable for now -- there's currently no graceful cleanup
6220          * called on exit().
6221          */
6222         (void) pthread_mutex_unlock(&gu->gu_freeze_lock);
6223         scf_handle_destroy(h);
6224 
6225         return (NULL);
6226 }
6227 
6228 
6229 /*
6230  * int next_action()
6231  *   Given an array of timestamps 'a' with 'num' elements, find the
6232  *   lowest non-zero timestamp and return its index. If there are no
6233  *   non-zero elements, return -1.
6234  */
6235 static int
6236 next_action(hrtime_t *a, int num)
6237 {
6238         hrtime_t t = 0;
6239         int i = 0, smallest = -1;
6240 
6241         for (i = 0; i < num; i++) {
6242                 if (t == 0) {
6243                         t = a[i];
6244                         smallest = i;
6245                 } else if (a[i] != 0 && a[i] < t) {
6246                         t = a[i];
6247                         smallest = i;
6248                 }
6249         }
6250 
6251         if (t == 0)
6252                 return (-1);
6253         else
6254                 return (smallest);
6255 }
6256 
6257 /*
6258  * void process_actions()
6259  *   Process actions requested by the administrator. Possibilities include:
6260  *   refresh, restart, maintenance mode off, maintenance mode on,
6261  *   maintenance mode immediate, and degraded.
6262  *
6263  *   The set of pending actions is represented in the repository as a
6264  *   per-instance property group, with each action being a single property
6265  *   in that group.  This property group is converted to an array, with each
6266  *   action type having an array slot.  The actions in the array at the
6267  *   time process_actions() is called are acted on in the order of the
6268  *   timestamp (which is the value stored in the slot).  A value of zero
6269  *   indicates that there is no pending action of the type associated with
6270  *   a particular slot.
6271  *
6272  *   Sending an action event multiple times before the restarter has a
6273  *   chance to process that action will force it to be run at the last
6274  *   timestamp where it appears in the ordering.
6275  *
6276  *   Turning maintenance mode on trumps all other actions.
6277  *
6278  *   Returns 0 or ECONNABORTED.
6279  */
6280 static int
6281 process_actions(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst)
6282 {
6283         scf_property_t *prop = NULL;
6284         scf_value_t *val = NULL;
6285         scf_type_t type;
6286         graph_vertex_t *vertex;
6287         admin_action_t a;
6288         int i, ret = 0, r;
6289         hrtime_t action_ts[NACTIONS];
6290         char *inst_name;
6291 
6292         r = libscf_instance_get_fmri(inst, &inst_name);
6293         switch (r) {
6294         case 0:
6295                 break;
6296 
6297         case ECONNABORTED:
6298                 return (ECONNABORTED);
6299 
6300         case ECANCELED:
6301                 return (0);
6302 
6303         default:
6304                 bad_error("libscf_instance_get_fmri", r);
6305         }
6306 
6307         MUTEX_LOCK(&dgraph_lock);
6308 
6309         vertex = vertex_get_by_name(inst_name);
6310         if (vertex == NULL) {
6311                 MUTEX_UNLOCK(&dgraph_lock);
6312                 log_framework(LOG_DEBUG, "%s: Can't find graph vertex. "
6313                     "The instance must have been removed.\n", inst_name);
6314                 startd_free(inst_name, max_scf_fmri_size);
6315                 return (0);
6316         }
6317 
6318         prop = safe_scf_property_create(h);
6319         val = safe_scf_value_create(h);
6320 
6321         for (i = 0; i < NACTIONS; i++) {
6322                 if (scf_pg_get_property(pg, admin_actions[i], prop) != 0) {
6323                         switch (scf_error()) {
6324                         case SCF_ERROR_CONNECTION_BROKEN:
6325                         default:
6326                                 ret = ECONNABORTED;
6327                                 goto out;
6328 
6329                         case SCF_ERROR_DELETED:
6330                                 goto out;
6331 
6332                         case SCF_ERROR_NOT_FOUND:
6333                                 action_ts[i] = 0;
6334                                 continue;
6335 
6336                         case SCF_ERROR_HANDLE_MISMATCH:
6337                         case SCF_ERROR_INVALID_ARGUMENT:
6338                         case SCF_ERROR_NOT_SET:
6339                                 bad_error("scf_pg_get_property", scf_error());
6340                         }
6341                 }
6342 
6343                 if (scf_property_type(prop, &type) != 0) {
6344                         switch (scf_error()) {
6345                         case SCF_ERROR_CONNECTION_BROKEN:
6346                         default:
6347                                 ret = ECONNABORTED;
6348                                 goto out;
6349 
6350                         case SCF_ERROR_DELETED:
6351                                 action_ts[i] = 0;
6352                                 continue;
6353 
6354                         case SCF_ERROR_NOT_SET:
6355                                 bad_error("scf_property_type", scf_error());
6356                         }
6357                 }
6358 
6359                 if (type != SCF_TYPE_INTEGER) {
6360                         action_ts[i] = 0;
6361                         continue;
6362                 }
6363 
6364                 if (scf_property_get_value(prop, val) != 0) {
6365                         switch (scf_error()) {
6366                         case SCF_ERROR_CONNECTION_BROKEN:
6367                         default:
6368                                 ret = ECONNABORTED;
6369                                 goto out;
6370 
6371                         case SCF_ERROR_DELETED:
6372                                 goto out;
6373 
6374                         case SCF_ERROR_NOT_FOUND:
6375                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6376                                 action_ts[i] = 0;
6377                                 continue;
6378 
6379                         case SCF_ERROR_NOT_SET:
6380                         case SCF_ERROR_PERMISSION_DENIED:
6381                                 bad_error("scf_property_get_value",
6382                                     scf_error());
6383                         }
6384                 }
6385 
6386                 r = scf_value_get_integer(val, &action_ts[i]);
6387                 assert(r == 0);
6388         }
6389 
6390         a = ADMIN_EVENT_MAINT_ON_IMMEDIATE;
6391         if (action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ||
6392             action_ts[ADMIN_EVENT_MAINT_ON]) {
6393                 a = action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ?
6394                     ADMIN_EVENT_MAINT_ON_IMMEDIATE : ADMIN_EVENT_MAINT_ON;
6395 
6396                 vertex_send_event(vertex, admin_events[a]);
6397                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6398                 switch (r) {
6399                 case 0:
6400                 case EACCES:
6401                         break;
6402 
6403                 case ECONNABORTED:
6404                         ret = ECONNABORTED;
6405                         goto out;
6406 
6407                 case EPERM:
6408                         uu_die("Insufficient privilege.\n");
6409                         /* NOTREACHED */
6410 
6411                 default:
6412                         bad_error("libscf_unset_action", r);
6413                 }
6414         }
6415 
6416         while ((a = next_action(action_ts, NACTIONS)) != -1) {
6417                 log_framework(LOG_DEBUG,
6418                     "Graph: processing %s action for %s.\n", admin_actions[a],
6419                     inst_name);
6420 
6421                 if (a == ADMIN_EVENT_REFRESH) {
6422                         r = dgraph_refresh_instance(vertex, inst);
6423                         switch (r) {
6424                         case 0:
6425                         case ECANCELED:
6426                         case EINVAL:
6427                         case -1:
6428                                 break;
6429 
6430                         case ECONNABORTED:
6431                                 /* pg & inst are reset now, so just return. */
6432                                 ret = ECONNABORTED;
6433                                 goto out;
6434 
6435                         default:
6436                                 bad_error("dgraph_refresh_instance", r);
6437                         }
6438                 }
6439 
6440                 vertex_send_event(vertex, admin_events[a]);
6441 
6442                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6443                 switch (r) {
6444                 case 0:
6445                 case EACCES:
6446                         break;
6447 
6448                 case ECONNABORTED:
6449                         ret = ECONNABORTED;
6450                         goto out;
6451 
6452                 case EPERM:
6453                         uu_die("Insufficient privilege.\n");
6454                         /* NOTREACHED */
6455 
6456                 default:
6457                         bad_error("libscf_unset_action", r);
6458                 }
6459 
6460                 action_ts[a] = 0;
6461         }
6462 
6463 out:
6464         MUTEX_UNLOCK(&dgraph_lock);
6465 
6466         scf_property_destroy(prop);
6467         scf_value_destroy(val);
6468         startd_free(inst_name, max_scf_fmri_size);
6469         return (ret);
6470 }
6471 
6472 /*
6473  * inst and pg_name are scratch space, and are unset on entry.
6474  * Returns
6475  *   0 - success
6476  *   ECONNRESET - success, but repository handle rebound
6477  *   ECONNABORTED - repository connection broken
6478  */
6479 static int
6480 process_pg_event(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst,
6481     char *pg_name)
6482 {
6483         int r;
6484         scf_property_t *prop;
6485         scf_value_t *val;
6486         char *fmri;
6487         boolean_t rebound = B_FALSE, rebind_inst = B_FALSE;
6488 
6489         if (scf_pg_get_name(pg, pg_name, max_scf_value_size) < 0) {
6490                 switch (scf_error()) {
6491                 case SCF_ERROR_CONNECTION_BROKEN:
6492                 default:
6493                         return (ECONNABORTED);
6494 
6495                 case SCF_ERROR_DELETED:
6496                         return (0);
6497 
6498                 case SCF_ERROR_NOT_SET:
6499                         bad_error("scf_pg_get_name", scf_error());
6500                 }
6501         }
6502 
6503         if (strcmp(pg_name, SCF_PG_GENERAL) == 0 ||
6504             strcmp(pg_name, SCF_PG_GENERAL_OVR) == 0) {
6505                 r = dgraph_update_general(pg);
6506                 switch (r) {
6507                 case 0:
6508                 case ENOTSUP:
6509                 case ECANCELED:
6510                         return (0);
6511 
6512                 case ECONNABORTED:
6513                         return (ECONNABORTED);
6514 
6515                 case -1:
6516                         /* Error should have been logged. */
6517                         return (0);
6518 
6519                 default:
6520                         bad_error("dgraph_update_general", r);
6521                 }
6522         } else if (strcmp(pg_name, SCF_PG_RESTARTER_ACTIONS) == 0) {
6523                 if (scf_pg_get_parent_instance(pg, inst) != 0) {
6524                         switch (scf_error()) {
6525                         case SCF_ERROR_CONNECTION_BROKEN:
6526                                 return (ECONNABORTED);
6527 
6528                         case SCF_ERROR_DELETED:
6529                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6530                                 /* Ignore commands on services. */
6531                                 return (0);
6532 
6533                         case SCF_ERROR_NOT_BOUND:
6534                         case SCF_ERROR_HANDLE_MISMATCH:
6535                         case SCF_ERROR_NOT_SET:
6536                         default:
6537                                 bad_error("scf_pg_get_parent_instance",
6538                                     scf_error());
6539                         }
6540                 }
6541 
6542                 return (process_actions(h, pg, inst));
6543         }
6544 
6545         if (strcmp(pg_name, SCF_PG_OPTIONS) != 0 &&
6546             strcmp(pg_name, SCF_PG_OPTIONS_OVR) != 0)
6547                 return (0);
6548 
6549         /*
6550          * We only care about the options[_ovr] property groups of our own
6551          * instance, so get the fmri and compare.  Plus, once we know it's
6552          * correct, if the repository connection is broken we know exactly what
6553          * property group we were operating on, and can look it up again.
6554          */
6555         if (scf_pg_get_parent_instance(pg, inst) != 0) {
6556                 switch (scf_error()) {
6557                 case SCF_ERROR_CONNECTION_BROKEN:
6558                         return (ECONNABORTED);
6559 
6560                 case SCF_ERROR_DELETED:
6561                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6562                         return (0);
6563 
6564                 case SCF_ERROR_HANDLE_MISMATCH:
6565                 case SCF_ERROR_NOT_BOUND:
6566                 case SCF_ERROR_NOT_SET:
6567                 default:
6568                         bad_error("scf_pg_get_parent_instance",
6569                             scf_error());
6570                 }
6571         }
6572 
6573         switch (r = libscf_instance_get_fmri(inst, &fmri)) {
6574         case 0:
6575                 break;
6576 
6577         case ECONNABORTED:
6578                 return (ECONNABORTED);
6579 
6580         case ECANCELED:
6581                 return (0);
6582 
6583         default:
6584                 bad_error("libscf_instance_get_fmri", r);
6585         }
6586 
6587         if (strcmp(fmri, SCF_SERVICE_STARTD) != 0) {
6588                 startd_free(fmri, max_scf_fmri_size);
6589                 return (0);
6590         }
6591 
6592         /*
6593          * update the information events flag
6594          */
6595         if (strcmp(pg_name, SCF_PG_OPTIONS) == 0)
6596                 info_events_all = libscf_get_info_events_all(pg);
6597 
6598         prop = safe_scf_property_create(h);
6599         val = safe_scf_value_create(h);
6600 
6601         if (strcmp(pg_name, SCF_PG_OPTIONS_OVR) == 0) {
6602                 /* See if we need to set the runlevel. */
6603                 /* CONSTCOND */
6604                 if (0) {
6605 rebind_pg:
6606                         libscf_handle_rebind(h);
6607                         rebound = B_TRUE;
6608 
6609                         r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6610                         switch (r) {
6611                         case 0:
6612                                 break;
6613 
6614                         case ECONNABORTED:
6615                                 goto rebind_pg;
6616 
6617                         case ENOENT:
6618                                 goto out;
6619 
6620                         case EINVAL:
6621                         case ENOTSUP:
6622                                 bad_error("libscf_lookup_instance", r);
6623                         }
6624 
6625                         if (scf_instance_get_pg(inst, pg_name, pg) != 0) {
6626                                 switch (scf_error()) {
6627                                 case SCF_ERROR_DELETED:
6628                                 case SCF_ERROR_NOT_FOUND:
6629                                         goto out;
6630 
6631                                 case SCF_ERROR_CONNECTION_BROKEN:
6632                                         goto rebind_pg;
6633 
6634                                 case SCF_ERROR_HANDLE_MISMATCH:
6635                                 case SCF_ERROR_NOT_BOUND:
6636                                 case SCF_ERROR_NOT_SET:
6637                                 case SCF_ERROR_INVALID_ARGUMENT:
6638                                 default:
6639                                         bad_error("scf_instance_get_pg",
6640                                             scf_error());
6641                                 }
6642                         }
6643                 }
6644 
6645                 if (scf_pg_get_property(pg, "runlevel", prop) == 0) {
6646                         r = dgraph_set_runlevel(pg, prop);
6647                         switch (r) {
6648                         case ECONNRESET:
6649                                 rebound = B_TRUE;
6650                                 rebind_inst = B_TRUE;
6651                                 /* FALLTHROUGH */
6652 
6653                         case 0:
6654                                 break;
6655 
6656                         case ECONNABORTED:
6657                                 goto rebind_pg;
6658 
6659                         case ECANCELED:
6660                                 goto out;
6661 
6662                         default:
6663                                 bad_error("dgraph_set_runlevel", r);
6664                         }
6665                 } else {
6666                         switch (scf_error()) {
6667                         case SCF_ERROR_CONNECTION_BROKEN:
6668                         default:
6669                                 goto rebind_pg;
6670 
6671                         case SCF_ERROR_DELETED:
6672                                 goto out;
6673 
6674                         case SCF_ERROR_NOT_FOUND:
6675                                 break;
6676 
6677                         case SCF_ERROR_INVALID_ARGUMENT:
6678                         case SCF_ERROR_HANDLE_MISMATCH:
6679                         case SCF_ERROR_NOT_BOUND:
6680                         case SCF_ERROR_NOT_SET:
6681                                 bad_error("scf_pg_get_property", scf_error());
6682                         }
6683                 }
6684         }
6685 
6686         if (rebind_inst) {
6687 lookup_inst:
6688                 r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6689                 switch (r) {
6690                 case 0:
6691                         break;
6692 
6693                 case ECONNABORTED:
6694                         libscf_handle_rebind(h);
6695                         rebound = B_TRUE;
6696                         goto lookup_inst;
6697 
6698                 case ENOENT:
6699                         goto out;
6700 
6701                 case EINVAL:
6702                 case ENOTSUP:
6703                         bad_error("libscf_lookup_instance", r);
6704                 }
6705         }
6706 
6707         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6708         switch (r) {
6709         case 0:
6710                 break;
6711 
6712         case ECONNABORTED:
6713                 libscf_handle_rebind(h);
6714                 rebound = B_TRUE;
6715                 goto lookup_inst;
6716 
6717         case EINVAL:
6718                 log_error(LOG_NOTICE,
6719                     "%s/%s property of %s is misconfigured.\n", pg_name,
6720                     SCF_PROPERTY_MILESTONE, SCF_SERVICE_STARTD);
6721                 /* FALLTHROUGH */
6722 
6723         case ECANCELED:
6724         case ENOENT:
6725                 (void) strcpy(fmri, "all");
6726                 break;
6727 
6728         default:
6729                 bad_error("libscf_get_milestone", r);
6730         }
6731 
6732         r = dgraph_set_milestone(fmri, h, B_FALSE);
6733         switch (r) {
6734         case 0:
6735         case ECONNRESET:
6736         case EALREADY:
6737                 break;
6738 
6739         case EINVAL:
6740                 log_error(LOG_WARNING, "Milestone %s is invalid.\n", fmri);
6741                 break;
6742 
6743         case ENOENT:
6744                 log_error(LOG_WARNING, "Milestone %s does not exist.\n", fmri);
6745                 break;
6746 
6747         default:
6748                 bad_error("dgraph_set_milestone", r);
6749         }
6750 
6751 out:
6752         startd_free(fmri, max_scf_fmri_size);
6753         scf_value_destroy(val);
6754         scf_property_destroy(prop);
6755 
6756         return (rebound ? ECONNRESET : 0);
6757 }
6758 
6759 /*
6760  * process_delete() deletes an instance from the dgraph if 'fmri' is an
6761  * instance fmri or if 'fmri' matches the 'general' property group of an
6762  * instance (or the 'general/enabled' property).
6763  *
6764  * 'fmri' may be overwritten and cannot be trusted on return by the caller.
6765  */
6766 static void
6767 process_delete(char *fmri, scf_handle_t *h)
6768 {
6769         char *lfmri, *end_inst_fmri;
6770         const char *inst_name = NULL;
6771         const char *pg_name = NULL;
6772         const char *prop_name = NULL;
6773 
6774         lfmri = safe_strdup(fmri);
6775 
6776         /* Determine if the FMRI is a property group or instance */
6777         if (scf_parse_svc_fmri(lfmri, NULL, NULL, &inst_name, &pg_name,
6778             &prop_name) != SCF_SUCCESS) {
6779                 log_error(LOG_WARNING,
6780                     "Received invalid FMRI \"%s\" from repository server.\n",
6781                     fmri);
6782         } else if (inst_name != NULL && pg_name == NULL) {
6783                 (void) dgraph_remove_instance(fmri, h);
6784         } else if (inst_name != NULL && pg_name != NULL) {
6785                 /*
6786                  * If we're deleting the 'general' property group or
6787                  * 'general/enabled' property then the whole instance
6788                  * must be removed from the dgraph.
6789                  */
6790                 if (strcmp(pg_name, SCF_PG_GENERAL) != 0) {
6791                         free(lfmri);
6792                         return;
6793                 }
6794 
6795                 if (prop_name != NULL &&
6796                     strcmp(prop_name, SCF_PROPERTY_ENABLED) != 0) {
6797                         free(lfmri);
6798                         return;
6799                 }
6800 
6801                 /*
6802                  * Because the instance has already been deleted from the
6803                  * repository, we cannot use any scf_ functions to retrieve
6804                  * the instance FMRI however we can easily reconstruct it
6805                  * manually.
6806                  */
6807                 end_inst_fmri = strstr(fmri, SCF_FMRI_PROPERTYGRP_PREFIX);
6808                 if (end_inst_fmri == NULL)
6809                         bad_error("process_delete", 0);
6810 
6811                 end_inst_fmri[0] = '\0';
6812 
6813                 (void) dgraph_remove_instance(fmri, h);
6814         }
6815 
6816         free(lfmri);
6817 }
6818 
6819 /*ARGSUSED*/
6820 void *
6821 repository_event_thread(void *unused)
6822 {
6823         scf_handle_t *h;
6824         scf_propertygroup_t *pg;
6825         scf_instance_t *inst;
6826         char *fmri = startd_alloc(max_scf_fmri_size);
6827         char *pg_name = startd_alloc(max_scf_value_size);
6828         int r;
6829 
6830         h = libscf_handle_create_bound_loop();
6831 
6832         pg = safe_scf_pg_create(h);
6833         inst = safe_scf_instance_create(h);
6834 
6835 retry:
6836         if (_scf_notify_add_pgtype(h, SCF_GROUP_FRAMEWORK) != SCF_SUCCESS) {
6837                 if (scf_error() == SCF_ERROR_CONNECTION_BROKEN) {
6838                         libscf_handle_rebind(h);
6839                 } else {
6840                         log_error(LOG_WARNING,
6841                             "Couldn't set up repository notification "
6842                             "for property group type %s: %s\n",
6843                             SCF_GROUP_FRAMEWORK, scf_strerror(scf_error()));
6844 
6845                         (void) sleep(1);
6846                 }
6847 
6848                 goto retry;
6849         }
6850 
6851         /*CONSTCOND*/
6852         while (1) {
6853                 ssize_t res;
6854 
6855                 /* Note: fmri is only set on delete events. */
6856                 res = _scf_notify_wait(pg, fmri, max_scf_fmri_size);
6857                 if (res < 0) {
6858                         libscf_handle_rebind(h);
6859                         goto retry;
6860                 } else if (res == 0) {
6861                         /*
6862                          * property group modified.  inst and pg_name are
6863                          * pre-allocated scratch space.
6864                          */
6865                         if (scf_pg_update(pg) < 0) {
6866                                 switch (scf_error()) {
6867                                 case SCF_ERROR_DELETED:
6868                                         continue;
6869 
6870                                 case SCF_ERROR_CONNECTION_BROKEN:
6871                                         log_error(LOG_WARNING,
6872                                             "Lost repository event due to "
6873                                             "disconnection.\n");
6874                                         libscf_handle_rebind(h);
6875                                         goto retry;
6876 
6877                                 case SCF_ERROR_NOT_BOUND:
6878                                 case SCF_ERROR_NOT_SET:
6879                                 default:
6880                                         bad_error("scf_pg_update", scf_error());
6881                                 }
6882                         }
6883 
6884                         r = process_pg_event(h, pg, inst, pg_name);
6885                         switch (r) {
6886                         case 0:
6887                                 break;
6888 
6889                         case ECONNABORTED:
6890                                 log_error(LOG_WARNING, "Lost repository event "
6891                                     "due to disconnection.\n");
6892                                 libscf_handle_rebind(h);
6893                                 /* FALLTHROUGH */
6894 
6895                         case ECONNRESET:
6896                                 goto retry;
6897 
6898                         default:
6899                                 bad_error("process_pg_event", r);
6900                         }
6901                 } else {
6902                         /*
6903                          * Service, instance, or pg deleted.
6904                          * Don't trust fmri on return.
6905                          */
6906                         process_delete(fmri, h);
6907                 }
6908         }
6909 
6910         /*NOTREACHED*/
6911         return (NULL);
6912 }
6913 
6914 void
6915 graph_engine_start()
6916 {
6917         int err;
6918 
6919         (void) startd_thread_create(graph_thread, NULL);
6920 
6921         MUTEX_LOCK(&dgraph_lock);
6922         while (!initial_milestone_set) {
6923                 err = pthread_cond_wait(&initial_milestone_cv, &dgraph_lock);
6924                 assert(err == 0);
6925         }
6926         MUTEX_UNLOCK(&dgraph_lock);
6927 
6928         (void) startd_thread_create(repository_event_thread, NULL);
6929         (void) startd_thread_create(graph_event_thread, NULL);
6930 }