1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
  25  * Copyright 2016 Toomas Soome <tsoome@me.com>
  26  * Copyright 2017 RackTop Systems.
  27  */
  28 
  29 /*
  30  * graph.c - master restarter graph engine
  31  *
  32  *   The graph engine keeps a dependency graph of all service instances on the
  33  *   system, as recorded in the repository.  It decides when services should
  34  *   be brought up or down based on service states and dependencies and sends
  35  *   commands to restarters to effect any changes.  It also executes
  36  *   administrator commands sent by svcadm via the repository.
  37  *
  38  *   The graph is stored in uu_list_t *dgraph and its vertices are
  39  *   graph_vertex_t's, each of which has a name and an integer id unique to
  40  *   its name (see dict.c).  A vertex's type attribute designates the type
  41  *   of object it represents: GVT_INST for service instances, GVT_SVC for
  42  *   service objects (since service instances may depend on another service,
  43  *   rather than service instance), GVT_FILE for files (which services may
  44  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
  45  *   vertices are necessary because dependency lists may have particular
  46  *   grouping types (require any, require all, optional, or exclude) and
  47  *   event-propagation characteristics.
  48  *
  49  *   The initial graph is built by libscf_populate_graph() invoking
  50  *   dgraph_add_instance() for each instance in the repository.  The function
  51  *   adds a GVT_SVC vertex for the service if one does not already exist, adds
  52  *   a GVT_INST vertex named by the FMRI of the instance, and sets up the edges.
  53  *   The resulting web of vertices & edges associated with an instance's vertex
  54  *   includes
  55  *
  56  *     - an edge from the GVT_SVC vertex for the instance's service
  57  *
  58  *     - an edge to the GVT_INST vertex of the instance's resarter, if its
  59  *       restarter is not svc.startd
  60  *
  61  *     - edges from other GVT_INST vertices if the instance is a restarter
  62  *
  63  *     - for each dependency property group in the instance's "running"
  64  *       snapshot, an edge to a GVT_GROUP vertex named by the FMRI of the
  65  *       instance and the name of the property group
  66  *
  67  *     - for each value of the "entities" property in each dependency property
  68  *       group, an edge from the corresponding GVT_GROUP vertex to a
  69  *       GVT_INST, GVT_SVC, or GVT_FILE vertex
  70  *
  71  *     - edges from GVT_GROUP vertices for each dependent instance
  72  *
  73  *   After the edges are set up the vertex's GV_CONFIGURED flag is set.  If
  74  *   there are problems, or if a service is mentioned in a dependency but does
  75  *   not exist in the repository, the GV_CONFIGURED flag will be clear.
  76  *
  77  *   The graph and all of its vertices are protected by the dgraph_lock mutex.
  78  *   See restarter.c for more information.
  79  *
  80  *   The properties of an instance fall into two classes: immediate and
  81  *   snapshotted.  Immediate properties should have an immediate effect when
  82  *   changed.  Snapshotted properties should be read from a snapshot, so they
  83  *   only change when the snapshot changes.  The immediate properties used by
  84  *   the graph engine are general/enabled, general/restarter, and the properties
  85  *   in the restarter_actions property group.  Since they are immediate, they
  86  *   are not read out of a snapshot.  The snapshotted properties used by the
  87  *   graph engine are those in the property groups with type "dependency" and
  88  *   are read out of the "running" snapshot.  The "running" snapshot is created
  89  *   by the the graph engine as soon as possible, and it is updated, along with
  90  *   in-core copies of the data (dependency information for the graph engine) on
  91  *   receipt of the refresh command from svcadm.  In addition, the graph engine
  92  *   updates the "start" snapshot from the "running" snapshot whenever a service
  93  *   comes online.
  94  *
  95  *   When a DISABLE event is requested by the administrator, svc.startd shutdown
  96  *   the dependents first before shutting down the requested service.
  97  *   In graph_enable_by_vertex, we create a subtree that contains the dependent
  98  *   vertices by marking those vertices with the GV_TOOFFLINE flag. And we mark
  99  *   the vertex to disable with the GV_TODISABLE flag. Once the tree is created,
 100  *   we send the _ADMIN_DISABLE event to the leaves. The leaves will then
 101  *   transition from STATE_ONLINE/STATE_DEGRADED to STATE_OFFLINE/STATE_MAINT.
 102  *   In gt_enter_offline and gt_enter_maint if the vertex was in a subtree then
 103  *   we clear the GV_TOOFFLINE flag and walk the dependencies to offline the new
 104  *   exposed leaves. We do the same until we reach the last leaf (the one with
 105  *   the GV_TODISABLE flag). If the vertex to disable is also part of a larger
 106  *   subtree (eg. multiple DISABLE events on vertices in the same subtree) then
 107  *   once the first vertex is disabled (GV_TODISABLE flag is removed), we
 108  *   continue to propagate the offline event to the vertex's dependencies.
 109  *
 110  *
 111  * SMF state transition notifications
 112  *
 113  *   When an instance of a service managed by SMF changes state, svc.startd may
 114  *   publish a GPEC sysevent. All transitions to or from maintenance, a
 115  *   transition cause by a hardware error will generate an event.
 116  *   Other transitions will generate an event if there exist notification
 117  *   parameter for that transition. Notification parameters are stored in the
 118  *   SMF repository for the service/instance they refer to. System-wide
 119  *   notification parameters are stored in the global instance.
 120  *   svc.startd can be told to send events for all SMF state transitions despite
 121  *   of notification parameters by setting options/info_events_all to true in
 122  *   restarter:default
 123  *
 124  *   The set of transitions that generate events is cached in the
 125  *   dgraph_vertex_t gv_stn_tset for service/instance and in the global
 126  *   stn_global for the system-wide set. They are re-read when instances are
 127  *   refreshed.
 128  *
 129  *   The GPEC events published by svc.startd are consumed by fmd(1M). After
 130  *   processing these events, fmd(1M) publishes the processed events to
 131  *   notification agents. The notification agents read the notification
 132  *   parameters from the SMF repository through libscf(3LIB) interfaces and send
 133  *   the notification, or not, based on those parameters.
 134  *
 135  *   Subscription and publishing to the GPEC channels is done with the
 136  *   libfmevent(3LIB) wrappers fmev_[r]publish_*() and
 137  *   fmev_shdl_(un)subscribe().
 138  *
 139  */
 140 
 141 #include <sys/uadmin.h>
 142 #include <sys/wait.h>
 143 
 144 #include <assert.h>
 145 #include <errno.h>
 146 #include <fcntl.h>
 147 #include <fm/libfmevent.h>
 148 #include <libscf.h>
 149 #include <libscf_priv.h>
 150 #include <librestart.h>
 151 #include <libuutil.h>
 152 #include <locale.h>
 153 #include <poll.h>
 154 #include <pthread.h>
 155 #include <signal.h>
 156 #include <stddef.h>
 157 #include <stdio.h>
 158 #include <stdlib.h>
 159 #include <string.h>
 160 #include <strings.h>
 161 #include <sys/statvfs.h>
 162 #include <sys/uadmin.h>
 163 #include <zone.h>
 164 #if defined(__x86)
 165 #include <libbe.h>
 166 #endif  /* __x86 */
 167 
 168 #include "startd.h"
 169 #include "protocol.h"
 170 
 171 
 172 #define MILESTONE_NONE  ((graph_vertex_t *)1)
 173 
 174 #define CONSOLE_LOGIN_FMRI      "svc:/system/console-login:default"
 175 #define FS_MINIMAL_FMRI         "svc:/system/filesystem/minimal:default"
 176 
 177 #define VERTEX_REMOVED  0       /* vertex has been freed  */
 178 #define VERTEX_INUSE    1       /* vertex is still in use */
 179 
 180 #define IS_ENABLED(v) ((v)->gv_flags & (GV_ENABLED | GV_ENBLD_NOOVR))
 181 
 182 /*
 183  * stn_global holds the tset for the system wide notification parameters.
 184  * It is updated on refresh of svc:/system/svc/global:default
 185  *
 186  * There are two assumptions that relax the need for a mutex:
 187  *     1. 32-bit value assignments are atomic
 188  *     2. Its value is consumed only in one point at
 189  *     dgraph_state_transition_notify(). There are no test and set races.
 190  *
 191  *     If either assumption is broken, we'll need a mutex to synchronize
 192  *     access to stn_global
 193  */
 194 int32_t stn_global;
 195 /*
 196  * info_events_all holds a flag to override notification parameters and send
 197  * Information events for all state transitions.
 198  * same about the need of a mutex here.
 199  */
 200 int info_events_all;
 201 
 202 /*
 203  * Services in these states are not considered 'down' by the
 204  * milestone/shutdown code.
 205  */
 206 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
 207         (state) == RESTARTER_STATE_DEGRADED || \
 208         (state) == RESTARTER_STATE_OFFLINE)
 209 
 210 #define is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
 211         ((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \
 212         (v->gv_restart < RERR_RESTART)))
 213 
 214 #define is_inst_bypassed(v) ((v->gv_type == GVT_INST) && \
 215         ((v->gv_flags & GV_TODISABLE) || \
 216         (v->gv_flags & GV_TOOFFLINE)))
 217 
 218 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
 219 static uu_list_t *dgraph;
 220 static pthread_mutex_t dgraph_lock;
 221 
 222 /*
 223  * milestone indicates the current subgraph.  When NULL, it is the entire
 224  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
 225  * services on which the target vertex depends.
 226  */
 227 static graph_vertex_t *milestone = NULL;
 228 static boolean_t initial_milestone_set = B_FALSE;
 229 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
 230 
 231 /* protected by dgraph_lock */
 232 static boolean_t sulogin_thread_running = B_FALSE;
 233 static boolean_t sulogin_running = B_FALSE;
 234 static boolean_t console_login_ready = B_FALSE;
 235 
 236 /* Number of services to come down to complete milestone transition. */
 237 static uint_t non_subgraph_svcs;
 238 
 239 /*
 240  * These variables indicate what should be done when we reach the milestone
 241  * target milestone, i.e., when non_subgraph_svcs == 0.  They are acted upon in
 242  * dgraph_set_instance_state().
 243  */
 244 static int halting = -1;
 245 static boolean_t go_single_user_mode = B_FALSE;
 246 static boolean_t go_to_level1 = B_FALSE;
 247 
 248 /*
 249  * Tracks when we started halting.
 250  */
 251 static time_t halting_time = 0;
 252 
 253 /*
 254  * This tracks the legacy runlevel to ensure we signal init and manage
 255  * utmpx entries correctly.
 256  */
 257 static char current_runlevel = '\0';
 258 
 259 /* Number of single user threads currently running */
 260 static pthread_mutex_t single_user_thread_lock;
 261 static int single_user_thread_count = 0;
 262 
 263 /* Statistics for dependency cycle-checking */
 264 static u_longlong_t dep_inserts = 0;
 265 static u_longlong_t dep_cycle_ns = 0;
 266 static u_longlong_t dep_insert_ns = 0;
 267 
 268 
 269 static const char * const emsg_invalid_restarter =
 270         "Transitioning %s to maintenance, restarter FMRI %s is invalid "
 271         "(see 'svcs -xv' for details).\n";
 272 static const char * const console_login_fmri = CONSOLE_LOGIN_FMRI;
 273 static const char * const single_user_fmri = SCF_MILESTONE_SINGLE_USER;
 274 static const char * const multi_user_fmri = SCF_MILESTONE_MULTI_USER;
 275 static const char * const multi_user_svr_fmri = SCF_MILESTONE_MULTI_USER_SERVER;
 276 
 277 
 278 /*
 279  * These services define the system being "up".  If none of them can come
 280  * online, then we will run sulogin on the console.  Note that the install ones
 281  * are for the miniroot and when installing CDs after the first.  can_come_up()
 282  * does the decision making, and an sulogin_thread() runs sulogin, which can be
 283  * started by dgraph_set_instance_state() or single_user_thread().
 284  *
 285  * NOTE: can_come_up() relies on SCF_MILESTONE_SINGLE_USER being the first
 286  * entry, which is only used when booting_to_single_user (boot -s) is set.
 287  * This is because when doing a "boot -s", sulogin is started from specials.c
 288  * after milestone/single-user comes online, for backwards compatibility.
 289  * In this case, SCF_MILESTONE_SINGLE_USER needs to be part of up_svcs
 290  * to ensure sulogin will be spawned if milestone/single-user cannot be reached.
 291  */
 292 static const char * const up_svcs[] = {
 293         SCF_MILESTONE_SINGLE_USER,
 294         CONSOLE_LOGIN_FMRI,
 295         "svc:/system/install-setup:default",
 296         "svc:/system/install:default",
 297         NULL
 298 };
 299 
 300 /* This array must have an element for each non-NULL element of up_svcs[]. */
 301 static graph_vertex_t *up_svcs_p[] = { NULL, NULL, NULL, NULL };
 302 
 303 /* These are for seed repository magic.  See can_come_up(). */
 304 static const char * const manifest_import = SCF_INSTANCE_MI;
 305 static graph_vertex_t *manifest_import_p = NULL;
 306 
 307 
 308 static char target_milestone_as_runlevel(void);
 309 static void graph_runlevel_changed(char rl, int online);
 310 static int dgraph_set_milestone(const char *, scf_handle_t *, boolean_t);
 311 static boolean_t should_be_in_subgraph(graph_vertex_t *v);
 312 static int mark_subtree(graph_edge_t *, void *);
 313 static boolean_t insubtree_dependents_down(graph_vertex_t *);
 314 
 315 /*
 316  * graph_vertex_compare()
 317  *      This function can compare either int *id or * graph_vertex_t *gv
 318  *      values, as the vertex id is always the first element of a
 319  *      graph_vertex structure.
 320  */
 321 /* ARGSUSED */
 322 static int
 323 graph_vertex_compare(const void *lc_arg, const void *rc_arg, void *private)
 324 {
 325         int lc_id = ((const graph_vertex_t *)lc_arg)->gv_id;
 326         int rc_id = *(int *)rc_arg;
 327 
 328         if (lc_id > rc_id)
 329                 return (1);
 330         if (lc_id < rc_id)
 331                 return (-1);
 332         return (0);
 333 }
 334 
 335 void
 336 graph_init()
 337 {
 338         graph_edge_pool = startd_list_pool_create("graph_edges",
 339             sizeof (graph_edge_t), offsetof(graph_edge_t, ge_link), NULL,
 340             UU_LIST_POOL_DEBUG);
 341         assert(graph_edge_pool != NULL);
 342 
 343         graph_vertex_pool = startd_list_pool_create("graph_vertices",
 344             sizeof (graph_vertex_t), offsetof(graph_vertex_t, gv_link),
 345             graph_vertex_compare, UU_LIST_POOL_DEBUG);
 346         assert(graph_vertex_pool != NULL);
 347 
 348         (void) pthread_mutex_init(&dgraph_lock, &mutex_attrs);
 349         (void) pthread_mutex_init(&single_user_thread_lock, &mutex_attrs);
 350         dgraph = startd_list_create(graph_vertex_pool, NULL, UU_LIST_SORTED);
 351         assert(dgraph != NULL);
 352 
 353         if (!st->st_initial)
 354                 current_runlevel = utmpx_get_runlevel();
 355 
 356         log_framework(LOG_DEBUG, "Initialized graph\n");
 357 }
 358 
 359 static graph_vertex_t *
 360 vertex_get_by_name(const char *name)
 361 {
 362         int id;
 363 
 364         assert(MUTEX_HELD(&dgraph_lock));
 365 
 366         id = dict_lookup_byname(name);
 367         if (id == -1)
 368                 return (NULL);
 369 
 370         return (uu_list_find(dgraph, &id, NULL, NULL));
 371 }
 372 
 373 static graph_vertex_t *
 374 vertex_get_by_id(int id)
 375 {
 376         assert(MUTEX_HELD(&dgraph_lock));
 377 
 378         if (id == -1)
 379                 return (NULL);
 380 
 381         return (uu_list_find(dgraph, &id, NULL, NULL));
 382 }
 383 
 384 /*
 385  * Creates a new vertex with the given name, adds it to the graph, and returns
 386  * a pointer to it.  The graph lock must be held by this thread on entry.
 387  */
 388 static graph_vertex_t *
 389 graph_add_vertex(const char *name)
 390 {
 391         int id;
 392         graph_vertex_t *v;
 393         void *p;
 394         uu_list_index_t idx;
 395 
 396         assert(MUTEX_HELD(&dgraph_lock));
 397 
 398         id = dict_insert(name);
 399 
 400         v = startd_zalloc(sizeof (*v));
 401 
 402         v->gv_id = id;
 403 
 404         v->gv_name = startd_alloc(strlen(name) + 1);
 405         (void) strcpy(v->gv_name, name);
 406 
 407         v->gv_dependencies = startd_list_create(graph_edge_pool, v, 0);
 408         v->gv_dependents = startd_list_create(graph_edge_pool, v, 0);
 409 
 410         p = uu_list_find(dgraph, &id, NULL, &idx);
 411         assert(p == NULL);
 412 
 413         uu_list_node_init(v, &v->gv_link, graph_vertex_pool);
 414         uu_list_insert(dgraph, v, idx);
 415 
 416         return (v);
 417 }
 418 
 419 /*
 420  * Removes v from the graph and frees it.  The graph should be locked by this
 421  * thread, and v should have no edges associated with it.
 422  */
 423 static void
 424 graph_remove_vertex(graph_vertex_t *v)
 425 {
 426         assert(MUTEX_HELD(&dgraph_lock));
 427 
 428         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 429         assert(uu_list_numnodes(v->gv_dependents) == 0);
 430         assert(v->gv_refs == 0);
 431 
 432         startd_free(v->gv_name, strlen(v->gv_name) + 1);
 433         uu_list_destroy(v->gv_dependencies);
 434         uu_list_destroy(v->gv_dependents);
 435         uu_list_remove(dgraph, v);
 436 
 437         startd_free(v, sizeof (graph_vertex_t));
 438 }
 439 
 440 static void
 441 graph_add_edge(graph_vertex_t *fv, graph_vertex_t *tv)
 442 {
 443         graph_edge_t *e, *re;
 444         int r;
 445 
 446         assert(MUTEX_HELD(&dgraph_lock));
 447 
 448         e = startd_alloc(sizeof (graph_edge_t));
 449         re = startd_alloc(sizeof (graph_edge_t));
 450 
 451         e->ge_parent = fv;
 452         e->ge_vertex = tv;
 453 
 454         re->ge_parent = tv;
 455         re->ge_vertex = fv;
 456 
 457         uu_list_node_init(e, &e->ge_link, graph_edge_pool);
 458         r = uu_list_insert_before(fv->gv_dependencies, NULL, e);
 459         assert(r == 0);
 460 
 461         uu_list_node_init(re, &re->ge_link, graph_edge_pool);
 462         r = uu_list_insert_before(tv->gv_dependents, NULL, re);
 463         assert(r == 0);
 464 }
 465 
 466 static void
 467 graph_remove_edge(graph_vertex_t *v, graph_vertex_t *dv)
 468 {
 469         graph_edge_t *e;
 470 
 471         for (e = uu_list_first(v->gv_dependencies);
 472             e != NULL;
 473             e = uu_list_next(v->gv_dependencies, e)) {
 474                 if (e->ge_vertex == dv) {
 475                         uu_list_remove(v->gv_dependencies, e);
 476                         startd_free(e, sizeof (graph_edge_t));
 477                         break;
 478                 }
 479         }
 480 
 481         for (e = uu_list_first(dv->gv_dependents);
 482             e != NULL;
 483             e = uu_list_next(dv->gv_dependents, e)) {
 484                 if (e->ge_vertex == v) {
 485                         uu_list_remove(dv->gv_dependents, e);
 486                         startd_free(e, sizeof (graph_edge_t));
 487                         break;
 488                 }
 489         }
 490 }
 491 
 492 static void
 493 remove_inst_vertex(graph_vertex_t *v)
 494 {
 495         graph_edge_t *e;
 496         graph_vertex_t *sv;
 497         int i;
 498 
 499         assert(MUTEX_HELD(&dgraph_lock));
 500         assert(uu_list_numnodes(v->gv_dependents) == 1);
 501         assert(uu_list_numnodes(v->gv_dependencies) == 0);
 502         assert(v->gv_refs == 0);
 503         assert((v->gv_flags & GV_CONFIGURED) == 0);
 504 
 505         e = uu_list_first(v->gv_dependents);
 506         sv = e->ge_vertex;
 507         graph_remove_edge(sv, v);
 508 
 509         for (i = 0; up_svcs[i] != NULL; ++i) {
 510                 if (up_svcs_p[i] == v)
 511                         up_svcs_p[i] = NULL;
 512         }
 513 
 514         if (manifest_import_p == v)
 515                 manifest_import_p = NULL;
 516 
 517         graph_remove_vertex(v);
 518 
 519         if (uu_list_numnodes(sv->gv_dependencies) == 0 &&
 520             uu_list_numnodes(sv->gv_dependents) == 0 &&
 521             sv->gv_refs == 0)
 522                 graph_remove_vertex(sv);
 523 }
 524 
 525 static void
 526 graph_walk_dependents(graph_vertex_t *v, void (*func)(graph_vertex_t *, void *),
 527     void *arg)
 528 {
 529         graph_edge_t *e;
 530 
 531         for (e = uu_list_first(v->gv_dependents);
 532             e != NULL;
 533             e = uu_list_next(v->gv_dependents, e))
 534                 func(e->ge_vertex, arg);
 535 }
 536 
 537 static void
 538 graph_walk_dependencies(graph_vertex_t *v,
 539     void (*func)(graph_vertex_t *, void *), void *arg)
 540 {
 541         graph_edge_t *e;
 542 
 543         assert(MUTEX_HELD(&dgraph_lock));
 544 
 545         for (e = uu_list_first(v->gv_dependencies);
 546             e != NULL;
 547             e = uu_list_next(v->gv_dependencies, e)) {
 548 
 549                 func(e->ge_vertex, arg);
 550         }
 551 }
 552 
 553 /*
 554  * Generic graph walking function.
 555  *
 556  * Given a vertex, this function will walk either dependencies
 557  * (WALK_DEPENDENCIES) or dependents (WALK_DEPENDENTS) of a vertex recursively
 558  * for the entire graph.  It will avoid cycles and never visit the same vertex
 559  * twice.
 560  *
 561  * We avoid traversing exclusion dependencies, because they are allowed to
 562  * create cycles in the graph.  When propagating satisfiability, there is no
 563  * need to walk exclusion dependencies because exclude_all_satisfied() doesn't
 564  * test for satisfiability.
 565  *
 566  * The walker takes two callbacks.  The first is called before examining the
 567  * dependents of each vertex.  The second is called on each vertex after
 568  * examining its dependents.  This allows is_path_to() to construct a path only
 569  * after the target vertex has been found.
 570  */
 571 typedef enum {
 572         WALK_DEPENDENTS,
 573         WALK_DEPENDENCIES
 574 } graph_walk_dir_t;
 575 
 576 typedef int (*graph_walk_cb_t)(graph_vertex_t *, void *);
 577 
 578 typedef struct graph_walk_info {
 579         graph_walk_dir_t        gi_dir;
 580         uchar_t                 *gi_visited;    /* vertex bitmap */
 581         int                     (*gi_pre)(graph_vertex_t *, void *);
 582         void                    (*gi_post)(graph_vertex_t *, void *);
 583         void                    *gi_arg;        /* callback arg */
 584         int                     gi_ret;         /* return value */
 585 } graph_walk_info_t;
 586 
 587 static int
 588 graph_walk_recurse(graph_edge_t *e, graph_walk_info_t *gip)
 589 {
 590         uu_list_t *list;
 591         int r;
 592         graph_vertex_t *v = e->ge_vertex;
 593         int i;
 594         uint_t b;
 595 
 596         i = v->gv_id / 8;
 597         b = 1 << (v->gv_id % 8);
 598 
 599         /*
 600          * Check to see if we've visited this vertex already.
 601          */
 602         if (gip->gi_visited[i] & b)
 603                 return (UU_WALK_NEXT);
 604 
 605         gip->gi_visited[i] |= b;
 606 
 607         /*
 608          * Don't follow exclusions.
 609          */
 610         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
 611                 return (UU_WALK_NEXT);
 612 
 613         /*
 614          * Call pre-visit callback.  If this doesn't terminate the walk,
 615          * continue search.
 616          */
 617         if ((gip->gi_ret = gip->gi_pre(v, gip->gi_arg)) == UU_WALK_NEXT) {
 618                 /*
 619                  * Recurse using appropriate list.
 620                  */
 621                 if (gip->gi_dir == WALK_DEPENDENTS)
 622                         list = v->gv_dependents;
 623                 else
 624                         list = v->gv_dependencies;
 625 
 626                 r = uu_list_walk(list, (uu_walk_fn_t *)graph_walk_recurse,
 627                     gip, 0);
 628                 assert(r == 0);
 629         }
 630 
 631         /*
 632          * Callbacks must return either UU_WALK_NEXT or UU_WALK_DONE.
 633          */
 634         assert(gip->gi_ret == UU_WALK_NEXT || gip->gi_ret == UU_WALK_DONE);
 635 
 636         /*
 637          * If given a post-callback, call the function for every vertex.
 638          */
 639         if (gip->gi_post != NULL)
 640                 (void) gip->gi_post(v, gip->gi_arg);
 641 
 642         /*
 643          * Preserve the callback's return value.  If the callback returns
 644          * UU_WALK_DONE, then we propagate that to the caller in order to
 645          * terminate the walk.
 646          */
 647         return (gip->gi_ret);
 648 }
 649 
 650 static void
 651 graph_walk(graph_vertex_t *v, graph_walk_dir_t dir,
 652     int (*pre)(graph_vertex_t *, void *),
 653     void (*post)(graph_vertex_t *, void *), void *arg)
 654 {
 655         graph_walk_info_t gi;
 656         graph_edge_t fake;
 657         size_t sz = dictionary->dict_new_id / 8 + 1;
 658 
 659         gi.gi_visited = startd_zalloc(sz);
 660         gi.gi_pre = pre;
 661         gi.gi_post = post;
 662         gi.gi_arg = arg;
 663         gi.gi_dir = dir;
 664         gi.gi_ret = 0;
 665 
 666         /*
 667          * Fake up an edge for the first iteration
 668          */
 669         fake.ge_vertex = v;
 670         (void) graph_walk_recurse(&fake, &gi);
 671 
 672         startd_free(gi.gi_visited, sz);
 673 }
 674 
 675 typedef struct child_search {
 676         int     id;             /* id of vertex to look for */
 677         uint_t  depth;          /* recursion depth */
 678         /*
 679          * While the vertex is not found, path is NULL.  After the search, if
 680          * the vertex was found then path should point to a -1-terminated
 681          * array of vertex id's which constitute the path to the vertex.
 682          */
 683         int     *path;
 684 } child_search_t;
 685 
 686 static int
 687 child_pre(graph_vertex_t *v, void *arg)
 688 {
 689         child_search_t *cs = arg;
 690 
 691         cs->depth++;
 692 
 693         if (v->gv_id == cs->id) {
 694                 cs->path = startd_alloc((cs->depth + 1) * sizeof (int));
 695                 cs->path[cs->depth] = -1;
 696                 return (UU_WALK_DONE);
 697         }
 698 
 699         return (UU_WALK_NEXT);
 700 }
 701 
 702 static void
 703 child_post(graph_vertex_t *v, void *arg)
 704 {
 705         child_search_t *cs = arg;
 706 
 707         cs->depth--;
 708 
 709         if (cs->path != NULL)
 710                 cs->path[cs->depth] = v->gv_id;
 711 }
 712 
 713 /*
 714  * Look for a path from from to to.  If one exists, returns a pointer to
 715  * a NULL-terminated array of pointers to the vertices along the path.  If
 716  * there is no path, returns NULL.
 717  */
 718 static int *
 719 is_path_to(graph_vertex_t *from, graph_vertex_t *to)
 720 {
 721         child_search_t cs;
 722 
 723         cs.id = to->gv_id;
 724         cs.depth = 0;
 725         cs.path = NULL;
 726 
 727         graph_walk(from, WALK_DEPENDENCIES, child_pre, child_post, &cs);
 728 
 729         return (cs.path);
 730 }
 731 
 732 /*
 733  * Given an array of int's as returned by is_path_to, allocates a string of
 734  * their names joined by newlines.  Returns the size of the allocated buffer
 735  * in *sz and frees path.
 736  */
 737 static void
 738 path_to_str(int *path, char **cpp, size_t *sz)
 739 {
 740         int i;
 741         graph_vertex_t *v;
 742         size_t allocd, new_allocd;
 743         char *new, *name;
 744 
 745         assert(MUTEX_HELD(&dgraph_lock));
 746         assert(path[0] != -1);
 747 
 748         allocd = 1;
 749         *cpp = startd_alloc(1);
 750         (*cpp)[0] = '\0';
 751 
 752         for (i = 0; path[i] != -1; ++i) {
 753                 name = NULL;
 754 
 755                 v = vertex_get_by_id(path[i]);
 756 
 757                 if (v == NULL)
 758                         name = "<deleted>";
 759                 else if (v->gv_type == GVT_INST || v->gv_type == GVT_SVC)
 760                         name = v->gv_name;
 761 
 762                 if (name != NULL) {
 763                         new_allocd = allocd + strlen(name) + 1;
 764                         new = startd_alloc(new_allocd);
 765                         (void) strcpy(new, *cpp);
 766                         (void) strcat(new, name);
 767                         (void) strcat(new, "\n");
 768 
 769                         startd_free(*cpp, allocd);
 770 
 771                         *cpp = new;
 772                         allocd = new_allocd;
 773                 }
 774         }
 775 
 776         startd_free(path, sizeof (int) * (i + 1));
 777 
 778         *sz = allocd;
 779 }
 780 
 781 
 782 /*
 783  * This function along with run_sulogin() implements an exclusion relationship
 784  * between system/console-login and sulogin.  run_sulogin() will fail if
 785  * system/console-login is online, and the graph engine should call
 786  * graph_clogin_start() to bring system/console-login online, which defers the
 787  * start if sulogin is running.
 788  */
 789 static void
 790 graph_clogin_start(graph_vertex_t *v)
 791 {
 792         assert(MUTEX_HELD(&dgraph_lock));
 793 
 794         if (sulogin_running)
 795                 console_login_ready = B_TRUE;
 796         else
 797                 vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 798 }
 799 
 800 static void
 801 graph_su_start(graph_vertex_t *v)
 802 {
 803         /*
 804          * /etc/inittab used to have the initial /sbin/rcS as a 'sysinit'
 805          * entry with a runlevel of 'S', before jumping to the final
 806          * target runlevel (as set in initdefault).  We mimic that legacy
 807          * behavior here.
 808          */
 809         utmpx_set_runlevel('S', '0', B_FALSE);
 810         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
 811 }
 812 
 813 static void
 814 graph_post_su_online(void)
 815 {
 816         graph_runlevel_changed('S', 1);
 817 }
 818 
 819 static void
 820 graph_post_su_disable(void)
 821 {
 822         graph_runlevel_changed('S', 0);
 823 }
 824 
 825 static void
 826 graph_post_mu_online(void)
 827 {
 828         graph_runlevel_changed('2', 1);
 829 }
 830 
 831 static void
 832 graph_post_mu_disable(void)
 833 {
 834         graph_runlevel_changed('2', 0);
 835 }
 836 
 837 static void
 838 graph_post_mus_online(void)
 839 {
 840         graph_runlevel_changed('3', 1);
 841 }
 842 
 843 static void
 844 graph_post_mus_disable(void)
 845 {
 846         graph_runlevel_changed('3', 0);
 847 }
 848 
 849 static struct special_vertex_info {
 850         const char      *name;
 851         void            (*start_f)(graph_vertex_t *);
 852         void            (*post_online_f)(void);
 853         void            (*post_disable_f)(void);
 854 } special_vertices[] = {
 855         { CONSOLE_LOGIN_FMRI, graph_clogin_start, NULL, NULL },
 856         { SCF_MILESTONE_SINGLE_USER, graph_su_start,
 857             graph_post_su_online, graph_post_su_disable },
 858         { SCF_MILESTONE_MULTI_USER, NULL,
 859             graph_post_mu_online, graph_post_mu_disable },
 860         { SCF_MILESTONE_MULTI_USER_SERVER, NULL,
 861             graph_post_mus_online, graph_post_mus_disable },
 862         { NULL },
 863 };
 864 
 865 
 866 void
 867 vertex_send_event(graph_vertex_t *v, restarter_event_type_t e)
 868 {
 869         switch (e) {
 870         case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
 871                 assert(v->gv_state == RESTARTER_STATE_UNINIT);
 872 
 873                 MUTEX_LOCK(&st->st_load_lock);
 874                 st->st_load_instances++;
 875                 MUTEX_UNLOCK(&st->st_load_lock);
 876                 break;
 877 
 878         case RESTARTER_EVENT_TYPE_ENABLE:
 879                 log_framework(LOG_DEBUG, "Enabling %s.\n", v->gv_name);
 880                 assert(v->gv_state == RESTARTER_STATE_UNINIT ||
 881                     v->gv_state == RESTARTER_STATE_DISABLED ||
 882                     v->gv_state == RESTARTER_STATE_MAINT);
 883                 break;
 884 
 885         case RESTARTER_EVENT_TYPE_DISABLE:
 886         case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
 887                 log_framework(LOG_DEBUG, "Disabling %s.\n", v->gv_name);
 888                 assert(v->gv_state != RESTARTER_STATE_DISABLED);
 889                 break;
 890 
 891         case RESTARTER_EVENT_TYPE_STOP_RESET:
 892         case RESTARTER_EVENT_TYPE_STOP:
 893                 log_framework(LOG_DEBUG, "Stopping %s.\n", v->gv_name);
 894                 assert(v->gv_state == RESTARTER_STATE_DEGRADED ||
 895                     v->gv_state == RESTARTER_STATE_ONLINE);
 896                 break;
 897 
 898         case RESTARTER_EVENT_TYPE_START:
 899                 log_framework(LOG_DEBUG, "Starting %s.\n", v->gv_name);
 900                 assert(v->gv_state == RESTARTER_STATE_OFFLINE);
 901                 break;
 902 
 903         case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
 904         case RESTARTER_EVENT_TYPE_ADMIN_RESTORE:
 905         case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
 906         case RESTARTER_EVENT_TYPE_ADMIN_DEGRADE_IMMEDIATE:
 907         case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
 908         case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
 909         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
 910         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
 911         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
 912         case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
 913         case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
 914                 break;
 915 
 916         default:
 917 #ifndef NDEBUG
 918                 uu_warn("%s:%d: Bad event %d.\n", __FILE__, __LINE__, e);
 919 #endif
 920                 abort();
 921         }
 922 
 923         restarter_protocol_send_event(v->gv_name, v->gv_restarter_channel, e,
 924             v->gv_reason);
 925 }
 926 
 927 static void
 928 graph_unset_restarter(graph_vertex_t *v)
 929 {
 930         assert(MUTEX_HELD(&dgraph_lock));
 931         assert(v->gv_flags & GV_CONFIGURED);
 932 
 933         vertex_send_event(v, RESTARTER_EVENT_TYPE_REMOVE_INSTANCE);
 934 
 935         if (v->gv_restarter_id != -1) {
 936                 graph_vertex_t *rv;
 937 
 938                 rv = vertex_get_by_id(v->gv_restarter_id);
 939                 graph_remove_edge(v, rv);
 940         }
 941 
 942         v->gv_restarter_id = -1;
 943         v->gv_restarter_channel = NULL;
 944 }
 945 
 946 /*
 947  * Return VERTEX_REMOVED when the vertex passed in argument is deleted from the
 948  * dgraph otherwise return VERTEX_INUSE.
 949  */
 950 static int
 951 free_if_unrefed(graph_vertex_t *v)
 952 {
 953         assert(MUTEX_HELD(&dgraph_lock));
 954 
 955         if (v->gv_refs > 0)
 956                 return (VERTEX_INUSE);
 957 
 958         if (v->gv_type == GVT_SVC &&
 959             uu_list_numnodes(v->gv_dependents) == 0 &&
 960             uu_list_numnodes(v->gv_dependencies) == 0) {
 961                 graph_remove_vertex(v);
 962                 return (VERTEX_REMOVED);
 963         } else if (v->gv_type == GVT_INST &&
 964             (v->gv_flags & GV_CONFIGURED) == 0 &&
 965             uu_list_numnodes(v->gv_dependents) == 1 &&
 966             uu_list_numnodes(v->gv_dependencies) == 0) {
 967                 remove_inst_vertex(v);
 968                 return (VERTEX_REMOVED);
 969         }
 970 
 971         return (VERTEX_INUSE);
 972 }
 973 
 974 static void
 975 delete_depgroup(graph_vertex_t *v)
 976 {
 977         graph_edge_t *e;
 978         graph_vertex_t *dv;
 979 
 980         assert(MUTEX_HELD(&dgraph_lock));
 981         assert(v->gv_type == GVT_GROUP);
 982         assert(uu_list_numnodes(v->gv_dependents) == 0);
 983 
 984         while ((e = uu_list_first(v->gv_dependencies)) != NULL) {
 985                 dv = e->ge_vertex;
 986 
 987                 graph_remove_edge(v, dv);
 988 
 989                 switch (dv->gv_type) {
 990                 case GVT_INST:          /* instance dependency */
 991                 case GVT_SVC:           /* service dependency */
 992                         (void) free_if_unrefed(dv);
 993                         break;
 994 
 995                 case GVT_FILE:          /* file dependency */
 996                         assert(uu_list_numnodes(dv->gv_dependencies) == 0);
 997                         if (uu_list_numnodes(dv->gv_dependents) == 0)
 998                                 graph_remove_vertex(dv);
 999                         break;
1000 
1001                 default:
1002 #ifndef NDEBUG
1003                         uu_warn("%s:%d: Unexpected node type %d", __FILE__,
1004                             __LINE__, dv->gv_type);
1005 #endif
1006                         abort();
1007                 }
1008         }
1009 
1010         graph_remove_vertex(v);
1011 }
1012 
1013 static int
1014 delete_instance_deps_cb(graph_edge_t *e, void **ptrs)
1015 {
1016         graph_vertex_t *v = ptrs[0];
1017         boolean_t delete_restarter_dep = (boolean_t)ptrs[1];
1018         graph_vertex_t *dv;
1019 
1020         dv = e->ge_vertex;
1021 
1022         /*
1023          * We have four possibilities here:
1024          *   - GVT_INST: restarter
1025          *   - GVT_GROUP - GVT_INST: instance dependency
1026          *   - GVT_GROUP - GVT_SVC - GV_INST: service dependency
1027          *   - GVT_GROUP - GVT_FILE: file dependency
1028          */
1029         switch (dv->gv_type) {
1030         case GVT_INST:  /* restarter */
1031                 assert(dv->gv_id == v->gv_restarter_id);
1032                 if (delete_restarter_dep)
1033                         graph_remove_edge(v, dv);
1034                 break;
1035 
1036         case GVT_GROUP: /* pg dependency */
1037                 graph_remove_edge(v, dv);
1038                 delete_depgroup(dv);
1039                 break;
1040 
1041         case GVT_FILE:
1042                 /* These are currently not direct dependencies */
1043 
1044         default:
1045 #ifndef NDEBUG
1046                 uu_warn("%s:%d: Bad vertex type %d.\n", __FILE__, __LINE__,
1047                     dv->gv_type);
1048 #endif
1049                 abort();
1050         }
1051 
1052         return (UU_WALK_NEXT);
1053 }
1054 
1055 static void
1056 delete_instance_dependencies(graph_vertex_t *v, boolean_t delete_restarter_dep)
1057 {
1058         void *ptrs[2];
1059         int r;
1060 
1061         assert(MUTEX_HELD(&dgraph_lock));
1062         assert(v->gv_type == GVT_INST);
1063 
1064         ptrs[0] = v;
1065         ptrs[1] = (void *)delete_restarter_dep;
1066 
1067         r = uu_list_walk(v->gv_dependencies,
1068             (uu_walk_fn_t *)delete_instance_deps_cb, &ptrs, UU_WALK_ROBUST);
1069         assert(r == 0);
1070 }
1071 
1072 /*
1073  * int graph_insert_vertex_unconfigured()
1074  *   Insert a vertex without sending any restarter events. If the vertex
1075  *   already exists or creation is successful, return a pointer to it in *vp.
1076  *
1077  *   If type is not GVT_GROUP, dt can remain unset.
1078  *
1079  *   Returns 0, EEXIST, or EINVAL if the arguments are invalid (i.e., fmri
1080  *   doesn't agree with type, or type doesn't agree with dt).
1081  */
1082 static int
1083 graph_insert_vertex_unconfigured(const char *fmri, gv_type_t type,
1084     depgroup_type_t dt, restarter_error_t rt, graph_vertex_t **vp)
1085 {
1086         int r;
1087         int i;
1088 
1089         assert(MUTEX_HELD(&dgraph_lock));
1090 
1091         switch (type) {
1092         case GVT_SVC:
1093         case GVT_INST:
1094                 if (strncmp(fmri, "svc:", sizeof ("svc:") - 1) != 0)
1095                         return (EINVAL);
1096                 break;
1097 
1098         case GVT_FILE:
1099                 if (strncmp(fmri, "file:", sizeof ("file:") - 1) != 0)
1100                         return (EINVAL);
1101                 break;
1102 
1103         case GVT_GROUP:
1104                 if (dt <= 0 || rt < 0)
1105                         return (EINVAL);
1106                 break;
1107 
1108         default:
1109 #ifndef NDEBUG
1110                 uu_warn("%s:%d: Unknown type %d.\n", __FILE__, __LINE__, type);
1111 #endif
1112                 abort();
1113         }
1114 
1115         *vp = vertex_get_by_name(fmri);
1116         if (*vp != NULL)
1117                 return (EEXIST);
1118 
1119         *vp = graph_add_vertex(fmri);
1120 
1121         (*vp)->gv_type = type;
1122         (*vp)->gv_depgroup = dt;
1123         (*vp)->gv_restart = rt;
1124 
1125         (*vp)->gv_flags = 0;
1126         (*vp)->gv_state = RESTARTER_STATE_NONE;
1127 
1128         for (i = 0; special_vertices[i].name != NULL; ++i) {
1129                 if (strcmp(fmri, special_vertices[i].name) == 0) {
1130                         (*vp)->gv_start_f = special_vertices[i].start_f;
1131                         (*vp)->gv_post_online_f =
1132                             special_vertices[i].post_online_f;
1133                         (*vp)->gv_post_disable_f =
1134                             special_vertices[i].post_disable_f;
1135                         break;
1136                 }
1137         }
1138 
1139         (*vp)->gv_restarter_id = -1;
1140         (*vp)->gv_restarter_channel = 0;
1141 
1142         if (type == GVT_INST) {
1143                 char *sfmri;
1144                 graph_vertex_t *sv;
1145 
1146                 sfmri = inst_fmri_to_svc_fmri(fmri);
1147                 sv = vertex_get_by_name(sfmri);
1148                 if (sv == NULL) {
1149                         r = graph_insert_vertex_unconfigured(sfmri, GVT_SVC, 0,
1150                             0, &sv);
1151                         assert(r == 0);
1152                 }
1153                 startd_free(sfmri, max_scf_fmri_size);
1154 
1155                 graph_add_edge(sv, *vp);
1156         }
1157 
1158         /*
1159          * If this vertex is in the subgraph, mark it as so, for both
1160          * GVT_INST and GVT_SERVICE verteces.
1161          * A GVT_SERVICE vertex can only be in the subgraph if another instance
1162          * depends on it, in which case it's already been added to the graph
1163          * and marked as in the subgraph (by refresh_vertex()).  If a
1164          * GVT_SERVICE vertex was freshly added (by the code above), it means
1165          * that it has no dependents, and cannot be in the subgraph.
1166          * Regardless of this, we still check that gv_flags includes
1167          * GV_INSUBGRAPH in the event that future behavior causes the above
1168          * code to add a GVT_SERVICE vertex which should be in the subgraph.
1169          */
1170 
1171         (*vp)->gv_flags |= (should_be_in_subgraph(*vp)? GV_INSUBGRAPH : 0);
1172 
1173         return (0);
1174 }
1175 
1176 /*
1177  * Returns 0 on success or ELOOP if the dependency would create a cycle.
1178  */
1179 static int
1180 graph_insert_dependency(graph_vertex_t *fv, graph_vertex_t *tv, int **pathp)
1181 {
1182         hrtime_t now;
1183 
1184         assert(MUTEX_HELD(&dgraph_lock));
1185 
1186         /* cycle detection */
1187         now = gethrtime();
1188 
1189         /* Don't follow exclusions. */
1190         if (!(fv->gv_type == GVT_GROUP &&
1191             fv->gv_depgroup == DEPGRP_EXCLUDE_ALL)) {
1192                 *pathp = is_path_to(tv, fv);
1193                 if (*pathp)
1194                         return (ELOOP);
1195         }
1196 
1197         dep_cycle_ns += gethrtime() - now;
1198         ++dep_inserts;
1199         now = gethrtime();
1200 
1201         graph_add_edge(fv, tv);
1202 
1203         dep_insert_ns += gethrtime() - now;
1204 
1205         /* Check if the dependency adds the "to" vertex to the subgraph */
1206         tv->gv_flags |= (should_be_in_subgraph(tv) ? GV_INSUBGRAPH : 0);
1207 
1208         return (0);
1209 }
1210 
1211 static int
1212 inst_running(graph_vertex_t *v)
1213 {
1214         assert(v->gv_type == GVT_INST);
1215 
1216         if (v->gv_state == RESTARTER_STATE_ONLINE ||
1217             v->gv_state == RESTARTER_STATE_DEGRADED)
1218                 return (1);
1219 
1220         return (0);
1221 }
1222 
1223 /*
1224  * The dependency evaluation functions return
1225  *   1 - dependency satisfied
1226  *   0 - dependency unsatisfied
1227  *   -1 - dependency unsatisfiable (without administrator intervention)
1228  *
1229  * The functions also take a boolean satbility argument.  When true, the
1230  * functions may recurse in order to determine satisfiability.
1231  */
1232 static int require_any_satisfied(graph_vertex_t *, boolean_t);
1233 static int dependency_satisfied(graph_vertex_t *, boolean_t);
1234 
1235 /*
1236  * A require_all dependency is unsatisfied if any elements are unsatisfied.  It
1237  * is unsatisfiable if any elements are unsatisfiable.
1238  */
1239 static int
1240 require_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1241 {
1242         graph_edge_t *edge;
1243         int i;
1244         boolean_t any_unsatisfied;
1245 
1246         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1247                 return (1);
1248 
1249         any_unsatisfied = B_FALSE;
1250 
1251         for (edge = uu_list_first(groupv->gv_dependencies);
1252             edge != NULL;
1253             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1254                 i = dependency_satisfied(edge->ge_vertex, satbility);
1255                 if (i == 1)
1256                         continue;
1257 
1258                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1259                     "require_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1260                     edge->ge_vertex->gv_name, i == 0 ? "ed" : "able");
1261 
1262                 if (!satbility)
1263                         return (0);
1264 
1265                 if (i == -1)
1266                         return (-1);
1267 
1268                 any_unsatisfied = B_TRUE;
1269         }
1270 
1271         return (any_unsatisfied ? 0 : 1);
1272 }
1273 
1274 /*
1275  * A require_any dependency is satisfied if any element is satisfied.  It is
1276  * satisfiable if any element is satisfiable.
1277  */
1278 static int
1279 require_any_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1280 {
1281         graph_edge_t *edge;
1282         int s;
1283         boolean_t satisfiable;
1284 
1285         if (uu_list_numnodes(groupv->gv_dependencies) == 0)
1286                 return (1);
1287 
1288         satisfiable = B_FALSE;
1289 
1290         for (edge = uu_list_first(groupv->gv_dependencies);
1291             edge != NULL;
1292             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1293                 s = dependency_satisfied(edge->ge_vertex, satbility);
1294 
1295                 if (s == 1)
1296                         return (1);
1297 
1298                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1299                     "require_any(%s): %s is unsatisfi%s.\n",
1300                     groupv->gv_name, edge->ge_vertex->gv_name,
1301                     s == 0 ? "ed" : "able");
1302 
1303                 if (satbility && s == 0)
1304                         satisfiable = B_TRUE;
1305         }
1306 
1307         return ((!satbility || satisfiable) ? 0 : -1);
1308 }
1309 
1310 /*
1311  * An optional_all dependency only considers elements which are configured,
1312  * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1313  * is unsatisfied.
1314  *
1315  * Offline dependencies which are waiting for a dependency to come online are
1316  * unsatisfied.  Offline dependences which cannot possibly come online
1317  * (unsatisfiable) are always considered satisfied.
1318  */
1319 static int
1320 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1321 {
1322         graph_edge_t *edge;
1323         graph_vertex_t *v;
1324         boolean_t any_qualified;
1325         boolean_t any_unsatisfied;
1326         int i;
1327 
1328         any_qualified = B_FALSE;
1329         any_unsatisfied = B_FALSE;
1330 
1331         for (edge = uu_list_first(groupv->gv_dependencies);
1332             edge != NULL;
1333             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1334                 v = edge->ge_vertex;
1335 
1336                 switch (v->gv_type) {
1337                 case GVT_INST:
1338                         /* Skip missing instances */
1339                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1340                                 continue;
1341 
1342                         if (v->gv_state == RESTARTER_STATE_MAINT)
1343                                 continue;
1344 
1345                         any_qualified = B_TRUE;
1346                         if (v->gv_state == RESTARTER_STATE_OFFLINE ||
1347                             v->gv_state == RESTARTER_STATE_DISABLED) {
1348                                 /*
1349                                  * For offline/disabled dependencies,
1350                                  * treat unsatisfiable as satisfied.
1351                                  */
1352                                 i = dependency_satisfied(v, B_TRUE);
1353                                 if (i == -1)
1354                                         i = 1;
1355                         } else {
1356                                 i = dependency_satisfied(v, satbility);
1357                         }
1358                         break;
1359 
1360                 case GVT_FILE:
1361                         any_qualified = B_TRUE;
1362                         i = dependency_satisfied(v, satbility);
1363 
1364                         break;
1365 
1366                 case GVT_SVC: {
1367                         any_qualified = B_TRUE;
1368                         i = optional_all_satisfied(v, satbility);
1369 
1370                         break;
1371                 }
1372 
1373                 case GVT_GROUP:
1374                 default:
1375 #ifndef NDEBUG
1376                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1377                             __LINE__, v->gv_type);
1378 #endif
1379                         abort();
1380                 }
1381 
1382                 if (i == 1)
1383                         continue;
1384 
1385                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1386                     "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1387                     v->gv_name, i == 0 ? "ed" : "able");
1388 
1389                 if (!satbility)
1390                         return (0);
1391                 if (i == -1)
1392                         return (-1);
1393                 any_unsatisfied = B_TRUE;
1394         }
1395 
1396         if (!any_qualified)
1397                 return (1);
1398 
1399         return (any_unsatisfied ? 0 : 1);
1400 }
1401 
1402 /*
1403  * An exclude_all dependency is unsatisfied if any non-service element is
1404  * satisfied or any service instance which is configured, enabled, and not in
1405  * maintenance is satisfied.  Usually when unsatisfied, it is also
1406  * unsatisfiable.
1407  */
1408 #define LOG_EXCLUDE(u, v)                                               \
1409         log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,                   \
1410             "exclude_all(%s): %s is satisfied.\n",                      \
1411             (u)->gv_name, (v)->gv_name)
1412 
1413 /* ARGSUSED */
1414 static int
1415 exclude_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1416 {
1417         graph_edge_t *edge, *e2;
1418         graph_vertex_t *v, *v2;
1419 
1420         for (edge = uu_list_first(groupv->gv_dependencies);
1421             edge != NULL;
1422             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1423                 v = edge->ge_vertex;
1424 
1425                 switch (v->gv_type) {
1426                 case GVT_INST:
1427                         if ((v->gv_flags & GV_CONFIGURED) == 0)
1428                                 continue;
1429 
1430                         switch (v->gv_state) {
1431                         case RESTARTER_STATE_ONLINE:
1432                         case RESTARTER_STATE_DEGRADED:
1433                                 LOG_EXCLUDE(groupv, v);
1434                                 return (v->gv_flags & GV_ENABLED ? -1 : 0);
1435 
1436                         case RESTARTER_STATE_OFFLINE:
1437                         case RESTARTER_STATE_UNINIT:
1438                                 LOG_EXCLUDE(groupv, v);
1439                                 return (0);
1440 
1441                         case RESTARTER_STATE_DISABLED:
1442                         case RESTARTER_STATE_MAINT:
1443                                 continue;
1444 
1445                         default:
1446 #ifndef NDEBUG
1447                                 uu_warn("%s:%d: Unexpected vertex state %d.\n",
1448                                     __FILE__, __LINE__, v->gv_state);
1449 #endif
1450                                 abort();
1451                         }
1452                         /* NOTREACHED */
1453 
1454                 case GVT_SVC:
1455                         break;
1456 
1457                 case GVT_FILE:
1458                         if (!file_ready(v))
1459                                 continue;
1460                         LOG_EXCLUDE(groupv, v);
1461                         return (-1);
1462 
1463                 case GVT_GROUP:
1464                 default:
1465 #ifndef NDEBUG
1466                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1467                             __LINE__, v->gv_type);
1468 #endif
1469                         abort();
1470                 }
1471 
1472                 /* v represents a service */
1473                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1474                         continue;
1475 
1476                 for (e2 = uu_list_first(v->gv_dependencies);
1477                     e2 != NULL;
1478                     e2 = uu_list_next(v->gv_dependencies, e2)) {
1479                         v2 = e2->ge_vertex;
1480                         assert(v2->gv_type == GVT_INST);
1481 
1482                         if ((v2->gv_flags & GV_CONFIGURED) == 0)
1483                                 continue;
1484 
1485                         switch (v2->gv_state) {
1486                         case RESTARTER_STATE_ONLINE:
1487                         case RESTARTER_STATE_DEGRADED:
1488                                 LOG_EXCLUDE(groupv, v2);
1489                                 return (v2->gv_flags & GV_ENABLED ? -1 : 0);
1490 
1491                         case RESTARTER_STATE_OFFLINE:
1492                         case RESTARTER_STATE_UNINIT:
1493                                 LOG_EXCLUDE(groupv, v2);
1494                                 return (0);
1495 
1496                         case RESTARTER_STATE_DISABLED:
1497                         case RESTARTER_STATE_MAINT:
1498                                 continue;
1499 
1500                         default:
1501 #ifndef NDEBUG
1502                                 uu_warn("%s:%d: Unexpected vertex type %d.\n",
1503                                     __FILE__, __LINE__, v2->gv_type);
1504 #endif
1505                                 abort();
1506                         }
1507                 }
1508         }
1509 
1510         return (1);
1511 }
1512 
1513 /*
1514  * int instance_satisfied()
1515  *   Determine if all the dependencies are satisfied for the supplied instance
1516  *   vertex. Return 1 if they are, 0 if they aren't, and -1 if they won't be
1517  *   without administrator intervention.
1518  */
1519 static int
1520 instance_satisfied(graph_vertex_t *v, boolean_t satbility)
1521 {
1522         assert(v->gv_type == GVT_INST);
1523         assert(!inst_running(v));
1524 
1525         return (require_all_satisfied(v, satbility));
1526 }
1527 
1528 /*
1529  * Decide whether v can satisfy a dependency.  v can either be a child of
1530  * a group vertex, or of an instance vertex.
1531  */
1532 static int
1533 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1534 {
1535         switch (v->gv_type) {
1536         case GVT_INST:
1537                 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1538                         if (v->gv_flags & GV_DEATHROW) {
1539                                 /*
1540                                  * A dependency on an instance with GV_DEATHROW
1541                                  * flag is always considered as satisfied.
1542                                  */
1543                                 return (1);
1544                         }
1545                         return (-1);
1546                 }
1547 
1548                 /*
1549                  * Vertices may be transitioning so we try to figure out if
1550                  * the end state is likely to satisfy the dependency instead
1551                  * of assuming the dependency is unsatisfied/unsatisfiable.
1552                  *
1553                  * Support for optional_all dependencies depends on us getting
1554                  * this right because unsatisfiable dependencies are treated
1555                  * as being satisfied.
1556                  */
1557                 switch (v->gv_state) {
1558                 case RESTARTER_STATE_ONLINE:
1559                 case RESTARTER_STATE_DEGRADED:
1560                         if (v->gv_flags & GV_TODISABLE)
1561                                 return (-1);
1562                         if (v->gv_flags & GV_TOOFFLINE)
1563                                 return (0);
1564                         return (1);
1565 
1566                 case RESTARTER_STATE_OFFLINE:
1567                         if (!satbility || v->gv_flags & GV_TODISABLE)
1568                                 return (satbility ? -1 : 0);
1569                         return (instance_satisfied(v, satbility) != -1 ?
1570                             0 : -1);
1571 
1572                 case RESTARTER_STATE_DISABLED:
1573                         if (!satbility || !(v->gv_flags & GV_ENABLED))
1574                                 return (satbility ? -1 : 0);
1575                         return (instance_satisfied(v, satbility) != -1 ?
1576                             0 : -1);
1577 
1578                 case RESTARTER_STATE_MAINT:
1579                         return (-1);
1580 
1581                 case RESTARTER_STATE_UNINIT:
1582                         return (0);
1583 
1584                 default:
1585 #ifndef NDEBUG
1586                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
1587                             __FILE__, __LINE__, v->gv_state);
1588 #endif
1589                         abort();
1590                         /* NOTREACHED */
1591                 }
1592 
1593         case GVT_SVC:
1594                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1595                         return (-1);
1596                 return (require_any_satisfied(v, satbility));
1597 
1598         case GVT_FILE:
1599                 /* i.e., we assume files will not be automatically generated */
1600                 return (file_ready(v) ? 1 : -1);
1601 
1602         case GVT_GROUP:
1603                 break;
1604 
1605         default:
1606 #ifndef NDEBUG
1607                 uu_warn("%s:%d: Unexpected node type %d.\n", __FILE__, __LINE__,
1608                     v->gv_type);
1609 #endif
1610                 abort();
1611                 /* NOTREACHED */
1612         }
1613 
1614         switch (v->gv_depgroup) {
1615         case DEPGRP_REQUIRE_ANY:
1616                 return (require_any_satisfied(v, satbility));
1617 
1618         case DEPGRP_REQUIRE_ALL:
1619                 return (require_all_satisfied(v, satbility));
1620 
1621         case DEPGRP_OPTIONAL_ALL:
1622                 return (optional_all_satisfied(v, satbility));
1623 
1624         case DEPGRP_EXCLUDE_ALL:
1625                 return (exclude_all_satisfied(v, satbility));
1626 
1627         default:
1628 #ifndef NDEBUG
1629                 uu_warn("%s:%d: Unknown dependency grouping %d.\n", __FILE__,
1630                     __LINE__, v->gv_depgroup);
1631 #endif
1632                 abort();
1633         }
1634 }
1635 
1636 void
1637 graph_start_if_satisfied(graph_vertex_t *v)
1638 {
1639         if (v->gv_state == RESTARTER_STATE_OFFLINE &&
1640             instance_satisfied(v, B_FALSE) == 1) {
1641                 if (v->gv_start_f == NULL)
1642                         vertex_send_event(v, RESTARTER_EVENT_TYPE_START);
1643                 else
1644                         v->gv_start_f(v);
1645         }
1646 }
1647 
1648 /*
1649  * propagate_satbility()
1650  *
1651  * This function is used when the given vertex changes state in such a way that
1652  * one of its dependents may become unsatisfiable.  This happens when an
1653  * instance transitions between offline -> online, or from !running ->
1654  * maintenance, as well as when an instance is removed from the graph.
1655  *
1656  * We have to walk all the dependents, since optional_all dependencies several
1657  * levels up could become (un)satisfied, instead of unsatisfiable.  For example,
1658  *
1659  *      +-----+  optional_all  +-----+  require_all  +-----+
1660  *      |  A  |--------------->|  B  |-------------->|  C  |
1661  *      +-----+                +-----+               +-----+
1662  *
1663  *                                              offline -> maintenance
1664  *
1665  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1666  * an optional dependency, what was previously an unsatisfiable situation is now
1667  * satisfied (B will never come online, even though its state hasn't changed).
1668  *
1669  * Note that it's not necessary to continue examining dependents after reaching
1670  * an optional_all dependency.  It's not possible for an optional_all dependency
1671  * to change satisfiability without also coming online, in which case we get a
1672  * start event and propagation continues naturally.  However, it does no harm to
1673  * continue propagating satisfiability (as it is a relatively rare event), and
1674  * keeps the walker code simple and generic.
1675  */
1676 /*ARGSUSED*/
1677 static int
1678 satbility_cb(graph_vertex_t *v, void *arg)
1679 {
1680         if (is_inst_bypassed(v))
1681                 return (UU_WALK_NEXT);
1682 
1683         if (v->gv_type == GVT_INST)
1684                 graph_start_if_satisfied(v);
1685 
1686         return (UU_WALK_NEXT);
1687 }
1688 
1689 static void
1690 propagate_satbility(graph_vertex_t *v)
1691 {
1692         graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1693 }
1694 
1695 static void propagate_stop(graph_vertex_t *, void *);
1696 
1697 /*
1698  * propagate_start()
1699  *
1700  * This function is used to propagate a start event to the dependents of the
1701  * given vertex.  Any dependents that are offline but have their dependencies
1702  * satisfied are started.  Any dependents that are online and have restart_on
1703  * set to "restart" or "refresh" are restarted because their dependencies have
1704  * just changed.  This only happens with optional_all dependencies.
1705  */
1706 static void
1707 propagate_start(graph_vertex_t *v, void *arg)
1708 {
1709         restarter_error_t err = (restarter_error_t)arg;
1710 
1711         if (is_inst_bypassed(v))
1712                 return;
1713 
1714         switch (v->gv_type) {
1715         case GVT_INST:
1716                 /* Restarter */
1717                 if (inst_running(v)) {
1718                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1719                                 vertex_send_event(v,
1720                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1721                         }
1722                 } else {
1723                         graph_start_if_satisfied(v);
1724                 }
1725                 break;
1726 
1727         case GVT_GROUP:
1728                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1729                         graph_walk_dependents(v, propagate_stop,
1730                             (void *)RERR_RESTART);
1731                         break;
1732                 }
1733                 err = v->gv_restart;
1734                 /* FALLTHROUGH */
1735 
1736         case GVT_SVC:
1737                 graph_walk_dependents(v, propagate_start, (void *)err);
1738                 break;
1739 
1740         case GVT_FILE:
1741 #ifndef NDEBUG
1742                 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1743                     __FILE__, __LINE__);
1744 #endif
1745                 abort();
1746                 /* NOTREACHED */
1747 
1748         default:
1749 #ifndef NDEBUG
1750                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1751                     v->gv_type);
1752 #endif
1753                 abort();
1754         }
1755 }
1756 
1757 /*
1758  * propagate_stop()
1759  *
1760  * This function is used to propagate a stop event to the dependents of the
1761  * given vertex.  Any dependents that are online (or in degraded state) with
1762  * the restart_on property set to "restart" or "refresh" will be stopped as
1763  * their dependencies have just changed, propagate_start() will start them
1764  * again once their dependencies have been re-satisfied.
1765  */
1766 static void
1767 propagate_stop(graph_vertex_t *v, void *arg)
1768 {
1769         restarter_error_t err = (restarter_error_t)arg;
1770 
1771         if (is_inst_bypassed(v))
1772                 return;
1773 
1774         switch (v->gv_type) {
1775         case GVT_INST:
1776                 /* Restarter */
1777                 if (err > RERR_NONE && inst_running(v)) {
1778                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1779                                 vertex_send_event(v,
1780                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1781                         } else {
1782                                 vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1783                         }
1784                 }
1785                 break;
1786 
1787         case GVT_SVC:
1788                 graph_walk_dependents(v, propagate_stop, arg);
1789                 break;
1790 
1791         case GVT_FILE:
1792 #ifndef NDEBUG
1793                 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1794                     __FILE__, __LINE__);
1795 #endif
1796                 abort();
1797                 /* NOTREACHED */
1798 
1799         case GVT_GROUP:
1800                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1801                         graph_walk_dependents(v, propagate_start,
1802                             (void *)RERR_NONE);
1803                         break;
1804                 }
1805 
1806                 if (err == RERR_NONE || err > v->gv_restart)
1807                         break;
1808 
1809                 graph_walk_dependents(v, propagate_stop, arg);
1810                 break;
1811 
1812         default:
1813 #ifndef NDEBUG
1814                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1815                     v->gv_type);
1816 #endif
1817                 abort();
1818         }
1819 }
1820 
1821 void
1822 offline_vertex(graph_vertex_t *v)
1823 {
1824         scf_handle_t *h = libscf_handle_create_bound_loop();
1825         scf_instance_t *scf_inst = safe_scf_instance_create(h);
1826         scf_propertygroup_t *pg = safe_scf_pg_create(h);
1827         restarter_instance_state_t state, next_state;
1828         int r;
1829 
1830         assert(v->gv_type == GVT_INST);
1831 
1832         if (scf_inst == NULL)
1833                 bad_error("safe_scf_instance_create", scf_error());
1834         if (pg == NULL)
1835                 bad_error("safe_scf_pg_create", scf_error());
1836 
1837         /* if the vertex is already going offline, return */
1838 rep_retry:
1839         if (scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, scf_inst, NULL,
1840             NULL, SCF_DECODE_FMRI_EXACT) != 0) {
1841                 switch (scf_error()) {
1842                 case SCF_ERROR_CONNECTION_BROKEN:
1843                         libscf_handle_rebind(h);
1844                         goto rep_retry;
1845 
1846                 case SCF_ERROR_NOT_FOUND:
1847                         scf_pg_destroy(pg);
1848                         scf_instance_destroy(scf_inst);
1849                         (void) scf_handle_unbind(h);
1850                         scf_handle_destroy(h);
1851                         return;
1852                 }
1853                 uu_die("Can't decode FMRI %s: %s\n", v->gv_name,
1854                     scf_strerror(scf_error()));
1855         }
1856 
1857         r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
1858         if (r != 0) {
1859                 switch (scf_error()) {
1860                 case SCF_ERROR_CONNECTION_BROKEN:
1861                         libscf_handle_rebind(h);
1862                         goto rep_retry;
1863 
1864                 case SCF_ERROR_NOT_SET:
1865                 case SCF_ERROR_NOT_FOUND:
1866                         scf_pg_destroy(pg);
1867                         scf_instance_destroy(scf_inst);
1868                         (void) scf_handle_unbind(h);
1869                         scf_handle_destroy(h);
1870                         return;
1871 
1872                 default:
1873                         bad_error("scf_instance_get_pg", scf_error());
1874                 }
1875         } else {
1876                 r = libscf_read_states(pg, &state, &next_state);
1877                 if (r == 0 && (next_state == RESTARTER_STATE_OFFLINE ||
1878                     next_state == RESTARTER_STATE_DISABLED)) {
1879                         log_framework(LOG_DEBUG,
1880                             "%s: instance is already going down.\n",
1881                             v->gv_name);
1882                         scf_pg_destroy(pg);
1883                         scf_instance_destroy(scf_inst);
1884                         (void) scf_handle_unbind(h);
1885                         scf_handle_destroy(h);
1886                         return;
1887                 }
1888         }
1889 
1890         scf_pg_destroy(pg);
1891         scf_instance_destroy(scf_inst);
1892         (void) scf_handle_unbind(h);
1893         scf_handle_destroy(h);
1894 
1895         vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP_RESET);
1896 }
1897 
1898 /*
1899  * void graph_enable_by_vertex()
1900  *   If admin is non-zero, this is an administrative request for change
1901  *   of the enabled property.  Thus, send the ADMIN_DISABLE rather than
1902  *   a plain DISABLE restarter event.
1903  */
1904 void
1905 graph_enable_by_vertex(graph_vertex_t *vertex, int enable, int admin)
1906 {
1907         graph_vertex_t *v;
1908         int r;
1909 
1910         assert(MUTEX_HELD(&dgraph_lock));
1911         assert((vertex->gv_flags & GV_CONFIGURED));
1912 
1913         vertex->gv_flags = (vertex->gv_flags & ~GV_ENABLED) |
1914             (enable ? GV_ENABLED : 0);
1915 
1916         if (enable) {
1917                 if (vertex->gv_state != RESTARTER_STATE_OFFLINE &&
1918                     vertex->gv_state != RESTARTER_STATE_DEGRADED &&
1919                     vertex->gv_state != RESTARTER_STATE_ONLINE) {
1920                         /*
1921                          * In case the vertex was notified to go down,
1922                          * but now can return online, clear the _TOOFFLINE
1923                          * and _TODISABLE flags.
1924                          */
1925                         vertex->gv_flags &= ~GV_TOOFFLINE;
1926                         vertex->gv_flags &= ~GV_TODISABLE;
1927 
1928                         vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ENABLE);
1929                 }
1930 
1931                 /*
1932                  * Wait for state update from restarter before sending _START or
1933                  * _STOP.
1934                  */
1935 
1936                 return;
1937         }
1938 
1939         if (vertex->gv_state == RESTARTER_STATE_DISABLED)
1940                 return;
1941 
1942         if (!admin) {
1943                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_DISABLE);
1944 
1945                 /*
1946                  * Wait for state update from restarter before sending _START or
1947                  * _STOP.
1948                  */
1949 
1950                 return;
1951         }
1952 
1953         /*
1954          * If it is a DISABLE event requested by the administrator then we are
1955          * offlining the dependents first.
1956          */
1957 
1958         /*
1959          * Set GV_TOOFFLINE for the services we are offlining. We cannot
1960          * clear the GV_TOOFFLINE bits from all the services because
1961          * other DISABLE events might be handled at the same time.
1962          */
1963         vertex->gv_flags |= GV_TOOFFLINE;
1964 
1965         /* remember which vertex to disable... */
1966         vertex->gv_flags |= GV_TODISABLE;
1967 
1968         log_framework(LOG_DEBUG, "Marking in-subtree vertices before "
1969             "disabling %s.\n", vertex->gv_name);
1970 
1971         /* set GV_TOOFFLINE for its dependents */
1972         r = uu_list_walk(vertex->gv_dependents, (uu_walk_fn_t *)mark_subtree,
1973             NULL, 0);
1974         assert(r == 0);
1975 
1976         /* disable the instance now if there is nothing else to offline */
1977         if (insubtree_dependents_down(vertex) == B_TRUE) {
1978                 vertex_send_event(vertex, RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
1979                 return;
1980         }
1981 
1982         /*
1983          * This loop is similar to the one used for the graph reversal shutdown
1984          * and could be improved in term of performance for the subtree reversal
1985          * disable case.
1986          */
1987         for (v = uu_list_first(dgraph); v != NULL;
1988             v = uu_list_next(dgraph, v)) {
1989                 /* skip the vertex we are disabling for now */
1990                 if (v == vertex)
1991                         continue;
1992 
1993                 if (v->gv_type != GVT_INST ||
1994                     (v->gv_flags & GV_CONFIGURED) == 0 ||
1995                     (v->gv_flags & GV_ENABLED) == 0 ||
1996                     (v->gv_flags & GV_TOOFFLINE) == 0)
1997                         continue;
1998 
1999                 if ((v->gv_state != RESTARTER_STATE_ONLINE) &&
2000                     (v->gv_state != RESTARTER_STATE_DEGRADED)) {
2001                         /* continue if there is nothing to offline */
2002                         continue;
2003                 }
2004 
2005                 /*
2006                  * Instances which are up need to come down before we're
2007                  * done, but we can only offline the leaves here. An
2008                  * instance is a leaf when all its dependents are down.
2009                  */
2010                 if (insubtree_dependents_down(v) == B_TRUE) {
2011                         log_framework(LOG_DEBUG, "Offlining in-subtree "
2012                             "instance %s for %s.\n",
2013                             v->gv_name, vertex->gv_name);
2014                         offline_vertex(v);
2015                 }
2016         }
2017 }
2018 
2019 static int configure_vertex(graph_vertex_t *, scf_instance_t *);
2020 
2021 /*
2022  * Set the restarter for v to fmri_arg.  That is, make sure a vertex for
2023  * fmri_arg exists, make v depend on it, and send _ADD_INSTANCE for v.  If
2024  * v is already configured and fmri_arg indicates the current restarter, do
2025  * nothing.  If v is configured and fmri_arg is a new restarter, delete v's
2026  * dependency on the restarter, send _REMOVE_INSTANCE for v, and set the new
2027  * restarter.  Returns 0 on success, EINVAL if the FMRI is invalid,
2028  * ECONNABORTED if the repository connection is broken, and ELOOP
2029  * if the dependency would create a cycle.  In the last case, *pathp will
2030  * point to a -1-terminated array of ids which compose the path from v to
2031  * restarter_fmri.
2032  */
2033 int
2034 graph_change_restarter(graph_vertex_t *v, const char *fmri_arg, scf_handle_t *h,
2035     int **pathp)
2036 {
2037         char *restarter_fmri = NULL;
2038         graph_vertex_t *rv;
2039         int err;
2040         int id;
2041 
2042         assert(MUTEX_HELD(&dgraph_lock));
2043 
2044         if (fmri_arg[0] != '\0') {
2045                 err = fmri_canonify(fmri_arg, &restarter_fmri, B_TRUE);
2046                 if (err != 0) {
2047                         assert(err == EINVAL);
2048                         return (err);
2049                 }
2050         }
2051 
2052         if (restarter_fmri == NULL ||
2053             strcmp(restarter_fmri, SCF_SERVICE_STARTD) == 0) {
2054                 if (v->gv_flags & GV_CONFIGURED) {
2055                         if (v->gv_restarter_id == -1) {
2056                                 if (restarter_fmri != NULL)
2057                                         startd_free(restarter_fmri,
2058                                             max_scf_fmri_size);
2059                                 return (0);
2060                         }
2061 
2062                         graph_unset_restarter(v);
2063                 }
2064 
2065                 /* Master restarter, nothing to do. */
2066                 v->gv_restarter_id = -1;
2067                 v->gv_restarter_channel = NULL;
2068                 vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2069                 return (0);
2070         }
2071 
2072         if (v->gv_flags & GV_CONFIGURED) {
2073                 id = dict_lookup_byname(restarter_fmri);
2074                 if (id != -1 && v->gv_restarter_id == id) {
2075                         startd_free(restarter_fmri, max_scf_fmri_size);
2076                         return (0);
2077                 }
2078 
2079                 graph_unset_restarter(v);
2080         }
2081 
2082         err = graph_insert_vertex_unconfigured(restarter_fmri, GVT_INST, 0,
2083             RERR_NONE, &rv);
2084         startd_free(restarter_fmri, max_scf_fmri_size);
2085         assert(err == 0 || err == EEXIST);
2086 
2087         if (rv->gv_delegate_initialized == 0) {
2088                 if ((rv->gv_delegate_channel = restarter_protocol_init_delegate(
2089                     rv->gv_name)) == NULL)
2090                         return (EINVAL);
2091                 rv->gv_delegate_initialized = 1;
2092         }
2093         v->gv_restarter_id = rv->gv_id;
2094         v->gv_restarter_channel = rv->gv_delegate_channel;
2095 
2096         err = graph_insert_dependency(v, rv, pathp);
2097         if (err != 0) {
2098                 assert(err == ELOOP);
2099                 return (ELOOP);
2100         }
2101 
2102         vertex_send_event(v, RESTARTER_EVENT_TYPE_ADD_INSTANCE);
2103 
2104         if (!(rv->gv_flags & GV_CONFIGURED)) {
2105                 scf_instance_t *inst;
2106 
2107                 err = libscf_fmri_get_instance(h, rv->gv_name, &inst);
2108                 switch (err) {
2109                 case 0:
2110                         err = configure_vertex(rv, inst);
2111                         scf_instance_destroy(inst);
2112                         switch (err) {
2113                         case 0:
2114                         case ECANCELED:
2115                                 break;
2116 
2117                         case ECONNABORTED:
2118                                 return (ECONNABORTED);
2119 
2120                         default:
2121                                 bad_error("configure_vertex", err);
2122                         }
2123                         break;
2124 
2125                 case ECONNABORTED:
2126                         return (ECONNABORTED);
2127 
2128                 case ENOENT:
2129                         break;
2130 
2131                 case ENOTSUP:
2132                         /*
2133                          * The fmri doesn't specify an instance - translate
2134                          * to EINVAL.
2135                          */
2136                         return (EINVAL);
2137 
2138                 case EINVAL:
2139                 default:
2140                         bad_error("libscf_fmri_get_instance", err);
2141                 }
2142         }
2143 
2144         return (0);
2145 }
2146 
2147 
2148 /*
2149  * Add all of the instances of the service named by fmri to the graph.
2150  * Returns
2151  *   0 - success
2152  *   ENOENT - service indicated by fmri does not exist
2153  *
2154  * In both cases *reboundp will be B_TRUE if the handle was rebound, or B_FALSE
2155  * otherwise.
2156  */
2157 static int
2158 add_service(const char *fmri, scf_handle_t *h, boolean_t *reboundp)
2159 {
2160         scf_service_t *svc;
2161         scf_instance_t *inst;
2162         scf_iter_t *iter;
2163         char *inst_fmri;
2164         int ret, r;
2165 
2166         *reboundp = B_FALSE;
2167 
2168         svc = safe_scf_service_create(h);
2169         inst = safe_scf_instance_create(h);
2170         iter = safe_scf_iter_create(h);
2171         inst_fmri = startd_alloc(max_scf_fmri_size);
2172 
2173 rebound:
2174         if (scf_handle_decode_fmri(h, fmri, NULL, svc, NULL, NULL, NULL,
2175             SCF_DECODE_FMRI_EXACT) != 0) {
2176                 switch (scf_error()) {
2177                 case SCF_ERROR_CONNECTION_BROKEN:
2178                 default:
2179                         libscf_handle_rebind(h);
2180                         *reboundp = B_TRUE;
2181                         goto rebound;
2182 
2183                 case SCF_ERROR_NOT_FOUND:
2184                         ret = ENOENT;
2185                         goto out;
2186 
2187                 case SCF_ERROR_INVALID_ARGUMENT:
2188                 case SCF_ERROR_CONSTRAINT_VIOLATED:
2189                 case SCF_ERROR_NOT_BOUND:
2190                 case SCF_ERROR_HANDLE_MISMATCH:
2191                         bad_error("scf_handle_decode_fmri", scf_error());
2192                 }
2193         }
2194 
2195         if (scf_iter_service_instances(iter, svc) != 0) {
2196                 switch (scf_error()) {
2197                 case SCF_ERROR_CONNECTION_BROKEN:
2198                 default:
2199                         libscf_handle_rebind(h);
2200                         *reboundp = B_TRUE;
2201                         goto rebound;
2202 
2203                 case SCF_ERROR_DELETED:
2204                         ret = ENOENT;
2205                         goto out;
2206 
2207                 case SCF_ERROR_HANDLE_MISMATCH:
2208                 case SCF_ERROR_NOT_BOUND:
2209                 case SCF_ERROR_NOT_SET:
2210                         bad_error("scf_iter_service_instances", scf_error());
2211                 }
2212         }
2213 
2214         for (;;) {
2215                 r = scf_iter_next_instance(iter, inst);
2216                 if (r == 0)
2217                         break;
2218                 if (r != 1) {
2219                         switch (scf_error()) {
2220                         case SCF_ERROR_CONNECTION_BROKEN:
2221                         default:
2222                                 libscf_handle_rebind(h);
2223                                 *reboundp = B_TRUE;
2224                                 goto rebound;
2225 
2226                         case SCF_ERROR_DELETED:
2227                                 ret = ENOENT;
2228                                 goto out;
2229 
2230                         case SCF_ERROR_HANDLE_MISMATCH:
2231                         case SCF_ERROR_NOT_BOUND:
2232                         case SCF_ERROR_NOT_SET:
2233                         case SCF_ERROR_INVALID_ARGUMENT:
2234                                 bad_error("scf_iter_next_instance",
2235                                     scf_error());
2236                         }
2237                 }
2238 
2239                 if (scf_instance_to_fmri(inst, inst_fmri, max_scf_fmri_size) <
2240                     0) {
2241                         switch (scf_error()) {
2242                         case SCF_ERROR_CONNECTION_BROKEN:
2243                                 libscf_handle_rebind(h);
2244                                 *reboundp = B_TRUE;
2245                                 goto rebound;
2246 
2247                         case SCF_ERROR_DELETED:
2248                                 continue;
2249 
2250                         case SCF_ERROR_NOT_BOUND:
2251                         case SCF_ERROR_NOT_SET:
2252                                 bad_error("scf_instance_to_fmri", scf_error());
2253                         }
2254                 }
2255 
2256                 r = dgraph_add_instance(inst_fmri, inst, B_FALSE);
2257                 switch (r) {
2258                 case 0:
2259                 case ECANCELED:
2260                         break;
2261 
2262                 case EEXIST:
2263                         continue;
2264 
2265                 case ECONNABORTED:
2266                         libscf_handle_rebind(h);
2267                         *reboundp = B_TRUE;
2268                         goto rebound;
2269 
2270                 case EINVAL:
2271                 default:
2272                         bad_error("dgraph_add_instance", r);
2273                 }
2274         }
2275 
2276         ret = 0;
2277 
2278 out:
2279         startd_free(inst_fmri, max_scf_fmri_size);
2280         scf_iter_destroy(iter);
2281         scf_instance_destroy(inst);
2282         scf_service_destroy(svc);
2283         return (ret);
2284 }
2285 
2286 struct depfmri_info {
2287         graph_vertex_t  *v;             /* GVT_GROUP vertex */
2288         gv_type_t       type;           /* type of dependency */
2289         const char      *inst_fmri;     /* FMRI of parental GVT_INST vert. */
2290         const char      *pg_name;       /* Name of dependency pg */
2291         scf_handle_t    *h;
2292         int             err;            /* return error code */
2293         int             **pathp;        /* return circular dependency path */
2294 };
2295 
2296 /*
2297  * Find or create a vertex for fmri and make info->v depend on it.
2298  * Returns
2299  *   0 - success
2300  *   nonzero - failure
2301  *
2302  * On failure, sets info->err to
2303  *   EINVAL - fmri is invalid
2304  *            fmri does not match info->type
2305  *   ELOOP - Adding the dependency creates a circular dependency.  *info->pathp
2306  *           will point to an array of the ids of the members of the cycle.
2307  *   ECONNABORTED - repository connection was broken
2308  *   ECONNRESET - succeeded, but repository connection was reset
2309  */
2310 static int
2311 process_dependency_fmri(const char *fmri, struct depfmri_info *info)
2312 {
2313         int err;
2314         graph_vertex_t *depgroup_v, *v;
2315         char *fmri_copy, *cfmri;
2316         size_t fmri_copy_sz;
2317         const char *scope, *service, *instance, *pg;
2318         scf_instance_t *inst;
2319         boolean_t rebound;
2320 
2321         assert(MUTEX_HELD(&dgraph_lock));
2322 
2323         /* Get or create vertex for FMRI */
2324         depgroup_v = info->v;
2325 
2326         if (strncmp(fmri, "file:", sizeof ("file:") - 1) == 0) {
2327                 if (info->type != GVT_FILE) {
2328                         log_framework(LOG_NOTICE,
2329                             "FMRI \"%s\" is not allowed for the \"%s\" "
2330                             "dependency's type of instance %s.\n", fmri,
2331                             info->pg_name, info->inst_fmri);
2332                         return (info->err = EINVAL);
2333                 }
2334 
2335                 err = graph_insert_vertex_unconfigured(fmri, info->type, 0,
2336                     RERR_NONE, &v);
2337                 switch (err) {
2338                 case 0:
2339                         break;
2340 
2341                 case EEXIST:
2342                         assert(v->gv_type == GVT_FILE);
2343                         break;
2344 
2345                 case EINVAL:            /* prevented above */
2346                 default:
2347                         bad_error("graph_insert_vertex_unconfigured", err);
2348                 }
2349         } else {
2350                 if (info->type != GVT_INST) {
2351                         log_framework(LOG_NOTICE,
2352                             "FMRI \"%s\" is not allowed for the \"%s\" "
2353                             "dependency's type of instance %s.\n", fmri,
2354                             info->pg_name, info->inst_fmri);
2355                         return (info->err = EINVAL);
2356                 }
2357 
2358                 /*
2359                  * We must canonify fmri & add a vertex for it.
2360                  */
2361                 fmri_copy_sz = strlen(fmri) + 1;
2362                 fmri_copy = startd_alloc(fmri_copy_sz);
2363                 (void) strcpy(fmri_copy, fmri);
2364 
2365                 /* Determine if the FMRI is a property group or instance */
2366                 if (scf_parse_svc_fmri(fmri_copy, &scope, &service,
2367                     &instance, &pg, NULL) != 0) {
2368                         startd_free(fmri_copy, fmri_copy_sz);
2369                         log_framework(LOG_NOTICE,
2370                             "Dependency \"%s\" of %s has invalid FMRI "
2371                             "\"%s\".\n", info->pg_name, info->inst_fmri,
2372                             fmri);
2373                         return (info->err = EINVAL);
2374                 }
2375 
2376                 if (service == NULL || pg != NULL) {
2377                         startd_free(fmri_copy, fmri_copy_sz);
2378                         log_framework(LOG_NOTICE,
2379                             "Dependency \"%s\" of %s does not designate a "
2380                             "service or instance.\n", info->pg_name,
2381                             info->inst_fmri);
2382                         return (info->err = EINVAL);
2383                 }
2384 
2385                 if (scope == NULL || strcmp(scope, SCF_SCOPE_LOCAL) == 0) {
2386                         cfmri = uu_msprintf("svc:/%s%s%s",
2387                             service, instance ? ":" : "", instance ? instance :
2388                             "");
2389                 } else {
2390                         cfmri = uu_msprintf("svc://%s/%s%s%s",
2391                             scope, service, instance ? ":" : "", instance ?
2392                             instance : "");
2393                 }
2394 
2395                 startd_free(fmri_copy, fmri_copy_sz);
2396 
2397                 err = graph_insert_vertex_unconfigured(cfmri, instance ?
2398                     GVT_INST : GVT_SVC, instance ? 0 : DEPGRP_REQUIRE_ANY,
2399                     RERR_NONE, &v);
2400                 uu_free(cfmri);
2401                 switch (err) {
2402                 case 0:
2403                         break;
2404 
2405                 case EEXIST:
2406                         /* Verify v. */
2407                         if (instance != NULL)
2408                                 assert(v->gv_type == GVT_INST);
2409                         else
2410                                 assert(v->gv_type == GVT_SVC);
2411                         break;
2412 
2413                 default:
2414                         bad_error("graph_insert_vertex_unconfigured", err);
2415                 }
2416         }
2417 
2418         /* Add dependency from depgroup_v to new vertex */
2419         info->err = graph_insert_dependency(depgroup_v, v, info->pathp);
2420         switch (info->err) {
2421         case 0:
2422                 break;
2423 
2424         case ELOOP:
2425                 return (ELOOP);
2426 
2427         default:
2428                 bad_error("graph_insert_dependency", info->err);
2429         }
2430 
2431         /* This must be after we insert the dependency, to avoid looping. */
2432         switch (v->gv_type) {
2433         case GVT_INST:
2434                 if ((v->gv_flags & GV_CONFIGURED) != 0)
2435                         break;
2436 
2437                 inst = safe_scf_instance_create(info->h);
2438 
2439                 rebound = B_FALSE;
2440 
2441 rebound:
2442                 err = libscf_lookup_instance(v->gv_name, inst);
2443                 switch (err) {
2444                 case 0:
2445                         err = configure_vertex(v, inst);
2446                         switch (err) {
2447                         case 0:
2448                         case ECANCELED:
2449                                 break;
2450 
2451                         case ECONNABORTED:
2452                                 libscf_handle_rebind(info->h);
2453                                 rebound = B_TRUE;
2454                                 goto rebound;
2455 
2456                         default:
2457                                 bad_error("configure_vertex", err);
2458                         }
2459                         break;
2460 
2461                 case ENOENT:
2462                         break;
2463 
2464                 case ECONNABORTED:
2465                         libscf_handle_rebind(info->h);
2466                         rebound = B_TRUE;
2467                         goto rebound;
2468 
2469                 case EINVAL:
2470                 case ENOTSUP:
2471                 default:
2472                         bad_error("libscf_fmri_get_instance", err);
2473                 }
2474 
2475                 scf_instance_destroy(inst);
2476 
2477                 if (rebound)
2478                         return (info->err = ECONNRESET);
2479                 break;
2480 
2481         case GVT_SVC:
2482                 (void) add_service(v->gv_name, info->h, &rebound);
2483                 if (rebound)
2484                         return (info->err = ECONNRESET);
2485         }
2486 
2487         return (0);
2488 }
2489 
2490 struct deppg_info {
2491         graph_vertex_t  *v;             /* GVT_INST vertex */
2492         int             err;            /* return error */
2493         int             **pathp;        /* return circular dependency path */
2494 };
2495 
2496 /*
2497  * Make info->v depend on a new GVT_GROUP node for this property group,
2498  * and then call process_dependency_fmri() for the values of the entity
2499  * property.  Return 0 on success, or if something goes wrong return nonzero
2500  * and set info->err to ECONNABORTED, EINVAL, or the error code returned by
2501  * process_dependency_fmri().
2502  */
2503 static int
2504 process_dependency_pg(scf_propertygroup_t *pg, struct deppg_info *info)
2505 {
2506         scf_handle_t *h;
2507         depgroup_type_t deptype;
2508         restarter_error_t rerr;
2509         struct depfmri_info linfo;
2510         char *fmri, *pg_name;
2511         size_t fmri_sz;
2512         graph_vertex_t *depgrp;
2513         scf_property_t *prop;
2514         int err;
2515         int empty;
2516         scf_error_t scferr;
2517         ssize_t len;
2518 
2519         assert(MUTEX_HELD(&dgraph_lock));
2520 
2521         h = scf_pg_handle(pg);
2522 
2523         pg_name = startd_alloc(max_scf_name_size);
2524 
2525         len = scf_pg_get_name(pg, pg_name, max_scf_name_size);
2526         if (len < 0) {
2527                 startd_free(pg_name, max_scf_name_size);
2528                 switch (scf_error()) {
2529                 case SCF_ERROR_CONNECTION_BROKEN:
2530                 default:
2531                         return (info->err = ECONNABORTED);
2532 
2533                 case SCF_ERROR_DELETED:
2534                         return (info->err = 0);
2535 
2536                 case SCF_ERROR_NOT_SET:
2537                         bad_error("scf_pg_get_name", scf_error());
2538                 }
2539         }
2540 
2541         /*
2542          * Skip over empty dependency groups.  Since dependency property
2543          * groups are updated atomically, they are either empty or
2544          * fully populated.
2545          */
2546         empty = depgroup_empty(h, pg);
2547         if (empty < 0) {
2548                 log_error(LOG_INFO,
2549                     "Error reading dependency group \"%s\" of %s: %s\n",
2550                     pg_name, info->v->gv_name, scf_strerror(scf_error()));
2551                 startd_free(pg_name, max_scf_name_size);
2552                 return (info->err = EINVAL);
2553 
2554         } else if (empty == 1) {
2555                 log_framework(LOG_DEBUG,
2556                     "Ignoring empty dependency group \"%s\" of %s\n",
2557                     pg_name, info->v->gv_name);
2558                 startd_free(pg_name, max_scf_name_size);
2559                 return (info->err = 0);
2560         }
2561 
2562         fmri_sz = strlen(info->v->gv_name) + 1 + len + 1;
2563         fmri = startd_alloc(fmri_sz);
2564 
2565         (void) snprintf(fmri, fmri_sz, "%s>%s", info->v->gv_name,
2566             pg_name);
2567 
2568         /* Validate the pg before modifying the graph */
2569         deptype = depgroup_read_grouping(h, pg);
2570         if (deptype == DEPGRP_UNSUPPORTED) {
2571                 log_error(LOG_INFO,
2572                     "Dependency \"%s\" of %s has an unknown grouping value.\n",
2573                     pg_name, info->v->gv_name);
2574                 startd_free(fmri, fmri_sz);
2575                 startd_free(pg_name, max_scf_name_size);
2576                 return (info->err = EINVAL);
2577         }
2578 
2579         rerr = depgroup_read_restart(h, pg);
2580         if (rerr == RERR_UNSUPPORTED) {
2581                 log_error(LOG_INFO,
2582                     "Dependency \"%s\" of %s has an unknown restart_on value."
2583                     "\n", pg_name, info->v->gv_name);
2584                 startd_free(fmri, fmri_sz);
2585                 startd_free(pg_name, max_scf_name_size);
2586                 return (info->err = EINVAL);
2587         }
2588 
2589         prop = safe_scf_property_create(h);
2590 
2591         if (scf_pg_get_property(pg, SCF_PROPERTY_ENTITIES, prop) != 0) {
2592                 scferr = scf_error();
2593                 scf_property_destroy(prop);
2594                 if (scferr == SCF_ERROR_DELETED) {
2595                         startd_free(fmri, fmri_sz);
2596                         startd_free(pg_name, max_scf_name_size);
2597                         return (info->err = 0);
2598                 } else if (scferr != SCF_ERROR_NOT_FOUND) {
2599                         startd_free(fmri, fmri_sz);
2600                         startd_free(pg_name, max_scf_name_size);
2601                         return (info->err = ECONNABORTED);
2602                 }
2603 
2604                 log_error(LOG_INFO,
2605                     "Dependency \"%s\" of %s is missing a \"%s\" property.\n",
2606                     pg_name, info->v->gv_name, SCF_PROPERTY_ENTITIES);
2607 
2608                 startd_free(fmri, fmri_sz);
2609                 startd_free(pg_name, max_scf_name_size);
2610 
2611                 return (info->err = EINVAL);
2612         }
2613 
2614         /* Create depgroup vertex for pg */
2615         err = graph_insert_vertex_unconfigured(fmri, GVT_GROUP, deptype,
2616             rerr, &depgrp);
2617         assert(err == 0);
2618         startd_free(fmri, fmri_sz);
2619 
2620         /* Add dependency from inst vertex to new vertex */
2621         err = graph_insert_dependency(info->v, depgrp, info->pathp);
2622         /* ELOOP can't happen because this should be a new vertex */
2623         assert(err == 0);
2624 
2625         linfo.v = depgrp;
2626         linfo.type = depgroup_read_scheme(h, pg);
2627         linfo.inst_fmri = info->v->gv_name;
2628         linfo.pg_name = pg_name;
2629         linfo.h = h;
2630         linfo.err = 0;
2631         linfo.pathp = info->pathp;
2632         err = walk_property_astrings(prop, (callback_t)process_dependency_fmri,
2633             &linfo);
2634 
2635         scf_property_destroy(prop);
2636         startd_free(pg_name, max_scf_name_size);
2637 
2638         switch (err) {
2639         case 0:
2640         case EINTR:
2641                 return (info->err = linfo.err);
2642 
2643         case ECONNABORTED:
2644         case EINVAL:
2645                 return (info->err = err);
2646 
2647         case ECANCELED:
2648                 return (info->err = 0);
2649 
2650         case ECONNRESET:
2651                 return (info->err = ECONNABORTED);
2652 
2653         default:
2654                 bad_error("walk_property_astrings", err);
2655                 /* NOTREACHED */
2656         }
2657 }
2658 
2659 /*
2660  * Build the dependency info for v from the repository.  Returns 0 on success,
2661  * ECONNABORTED on repository disconnection, EINVAL if the repository
2662  * configuration is invalid, and ELOOP if a dependency would cause a cycle.
2663  * In the last case, *pathp will point to a -1-terminated array of ids which
2664  * constitute the rest of the dependency cycle.
2665  */
2666 static int
2667 set_dependencies(graph_vertex_t *v, scf_instance_t *inst, int **pathp)
2668 {
2669         struct deppg_info info;
2670         int err;
2671         uint_t old_configured;
2672 
2673         assert(MUTEX_HELD(&dgraph_lock));
2674 
2675         /*
2676          * Mark the vertex as configured during dependency insertion to avoid
2677          * dependency cycles (which can appear in the graph if one of the
2678          * vertices is an exclusion-group).
2679          */
2680         old_configured = v->gv_flags & GV_CONFIGURED;
2681         v->gv_flags |= GV_CONFIGURED;
2682 
2683         info.err = 0;
2684         info.v = v;
2685         info.pathp = pathp;
2686 
2687         err = walk_dependency_pgs(inst, (callback_t)process_dependency_pg,
2688             &info);
2689 
2690         if (!old_configured)
2691                 v->gv_flags &= ~GV_CONFIGURED;
2692 
2693         switch (err) {
2694         case 0:
2695         case EINTR:
2696                 return (info.err);
2697 
2698         case ECONNABORTED:
2699                 return (ECONNABORTED);
2700 
2701         case ECANCELED:
2702                 /* Should get delete event, so return 0. */
2703                 return (0);
2704 
2705         default:
2706                 bad_error("walk_dependency_pgs", err);
2707                 /* NOTREACHED */
2708         }
2709 }
2710 
2711 
2712 static void
2713 handle_cycle(const char *fmri, int *path)
2714 {
2715         const char *cp;
2716         size_t sz;
2717 
2718         assert(MUTEX_HELD(&dgraph_lock));
2719 
2720         path_to_str(path, (char **)&cp, &sz);
2721 
2722         log_error(LOG_ERR, "Transitioning %s to maintenance "
2723             "because it completes a dependency cycle (see svcs -xv for "
2724             "details):\n%s", fmri ? fmri : "?", cp);
2725 
2726         startd_free((void *)cp, sz);
2727 }
2728 
2729 /*
2730  * Increment the vertex's reference count to prevent the vertex removal
2731  * from the dgraph.
2732  */
2733 static void
2734 vertex_ref(graph_vertex_t *v)
2735 {
2736         assert(MUTEX_HELD(&dgraph_lock));
2737 
2738         v->gv_refs++;
2739 }
2740 
2741 /*
2742  * Decrement the vertex's reference count and remove the vertex from
2743  * the dgraph when possible.
2744  *
2745  * Return VERTEX_REMOVED when the vertex has been removed otherwise
2746  * return VERTEX_INUSE.
2747  */
2748 static int
2749 vertex_unref(graph_vertex_t *v)
2750 {
2751         assert(MUTEX_HELD(&dgraph_lock));
2752         assert(v->gv_refs > 0);
2753 
2754         v->gv_refs--;
2755 
2756         return (free_if_unrefed(v));
2757 }
2758 
2759 /*
2760  * When run on the dependencies of a vertex, populates list with
2761  * graph_edge_t's which point to the service vertices or the instance
2762  * vertices (no GVT_GROUP nodes) on which the vertex depends.
2763  *
2764  * Increment the vertex's reference count once the vertex is inserted
2765  * in the list. The vertex won't be able to be deleted from the dgraph
2766  * while it is referenced.
2767  */
2768 static int
2769 append_svcs_or_insts(graph_edge_t *e, uu_list_t *list)
2770 {
2771         graph_vertex_t *v = e->ge_vertex;
2772         graph_edge_t *new;
2773         int r;
2774 
2775         switch (v->gv_type) {
2776         case GVT_INST:
2777         case GVT_SVC:
2778                 break;
2779 
2780         case GVT_GROUP:
2781                 r = uu_list_walk(v->gv_dependencies,
2782                     (uu_walk_fn_t *)append_svcs_or_insts, list, 0);
2783                 assert(r == 0);
2784                 return (UU_WALK_NEXT);
2785 
2786         case GVT_FILE:
2787                 return (UU_WALK_NEXT);
2788 
2789         default:
2790 #ifndef NDEBUG
2791                 uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
2792                     __LINE__, v->gv_type);
2793 #endif
2794                 abort();
2795         }
2796 
2797         new = startd_alloc(sizeof (*new));
2798         new->ge_vertex = v;
2799         uu_list_node_init(new, &new->ge_link, graph_edge_pool);
2800         r = uu_list_insert_before(list, NULL, new);
2801         assert(r == 0);
2802 
2803         /*
2804          * Because we are inserting the vertex in a list, we don't want
2805          * the vertex to be freed while the list is in use. In order to
2806          * achieve that, increment the vertex's reference count.
2807          */
2808         vertex_ref(v);
2809 
2810         return (UU_WALK_NEXT);
2811 }
2812 
2813 static boolean_t
2814 should_be_in_subgraph(graph_vertex_t *v)
2815 {
2816         graph_edge_t *e;
2817 
2818         if (v == milestone)
2819                 return (B_TRUE);
2820 
2821         /*
2822          * v is in the subgraph if any of its dependents are in the subgraph.
2823          * Except for EXCLUDE_ALL dependents.  And OPTIONAL dependents only
2824          * count if we're enabled.
2825          */
2826         for (e = uu_list_first(v->gv_dependents);
2827             e != NULL;
2828             e = uu_list_next(v->gv_dependents, e)) {
2829                 graph_vertex_t *dv = e->ge_vertex;
2830 
2831                 if (!(dv->gv_flags & GV_INSUBGRAPH))
2832                         continue;
2833 
2834                 /*
2835                  * Don't include instances that are optional and disabled.
2836                  */
2837                 if (v->gv_type == GVT_INST && dv->gv_type == GVT_SVC) {
2838 
2839                         int in = 0;
2840                         graph_edge_t *ee;
2841 
2842                         for (ee = uu_list_first(dv->gv_dependents);
2843                             ee != NULL;
2844                             ee = uu_list_next(dv->gv_dependents, ee)) {
2845 
2846                                 graph_vertex_t *ddv = e->ge_vertex;
2847 
2848                                 if (ddv->gv_type == GVT_GROUP &&
2849                                     ddv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2850                                         continue;
2851 
2852                                 if (ddv->gv_type == GVT_GROUP &&
2853                                     ddv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2854                                     !(v->gv_flags & GV_ENBLD_NOOVR))
2855                                         continue;
2856 
2857                                 in = 1;
2858                         }
2859                         if (!in)
2860                                 continue;
2861                 }
2862                 if (v->gv_type == GVT_INST &&
2863                     dv->gv_type == GVT_GROUP &&
2864                     dv->gv_depgroup == DEPGRP_OPTIONAL_ALL &&
2865                     !(v->gv_flags & GV_ENBLD_NOOVR))
2866                         continue;
2867 
2868                 /* Don't include excluded services and instances */
2869                 if (dv->gv_type == GVT_GROUP &&
2870                     dv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
2871                         continue;
2872 
2873                 return (B_TRUE);
2874         }
2875 
2876         return (B_FALSE);
2877 }
2878 
2879 /*
2880  * Ensures that GV_INSUBGRAPH is set properly for v and its descendents.  If
2881  * any bits change, manipulate the repository appropriately.  Returns 0 or
2882  * ECONNABORTED.
2883  */
2884 static int
2885 eval_subgraph(graph_vertex_t *v, scf_handle_t *h)
2886 {
2887         boolean_t old = (v->gv_flags & GV_INSUBGRAPH) != 0;
2888         boolean_t new;
2889         graph_edge_t *e;
2890         scf_instance_t *inst;
2891         int ret = 0, r;
2892 
2893         assert(milestone != NULL && milestone != MILESTONE_NONE);
2894 
2895         new = should_be_in_subgraph(v);
2896 
2897         if (new == old)
2898                 return (0);
2899 
2900         log_framework(LOG_DEBUG, new ? "Adding %s to the subgraph.\n" :
2901             "Removing %s from the subgraph.\n", v->gv_name);
2902 
2903         v->gv_flags = (v->gv_flags & ~GV_INSUBGRAPH) |
2904             (new ? GV_INSUBGRAPH : 0);
2905 
2906         if (v->gv_type == GVT_INST && (v->gv_flags & GV_CONFIGURED)) {
2907                 int err;
2908 
2909 get_inst:
2910                 err = libscf_fmri_get_instance(h, v->gv_name, &inst);
2911                 if (err != 0) {
2912                         switch (err) {
2913                         case ECONNABORTED:
2914                                 libscf_handle_rebind(h);
2915                                 ret = ECONNABORTED;
2916                                 goto get_inst;
2917 
2918                         case ENOENT:
2919                                 break;
2920 
2921                         case EINVAL:
2922                         case ENOTSUP:
2923                         default:
2924                                 bad_error("libscf_fmri_get_instance", err);
2925                         }
2926                 } else {
2927                         const char *f;
2928 
2929                         if (new) {
2930                                 err = libscf_delete_enable_ovr(inst);
2931                                 f = "libscf_delete_enable_ovr";
2932                         } else {
2933                                 err = libscf_set_enable_ovr(inst, 0);
2934                                 f = "libscf_set_enable_ovr";
2935                         }
2936                         scf_instance_destroy(inst);
2937                         switch (err) {
2938                         case 0:
2939                         case ECANCELED:
2940                                 break;
2941 
2942                         case ECONNABORTED:
2943                                 libscf_handle_rebind(h);
2944                                 /*
2945                                  * We must continue so the graph is updated,
2946                                  * but we must return ECONNABORTED so any
2947                                  * libscf state held by any callers is reset.
2948                                  */
2949                                 ret = ECONNABORTED;
2950                                 goto get_inst;
2951 
2952                         case EROFS:
2953                         case EPERM:
2954                                 log_error(LOG_WARNING,
2955                                     "Could not set %s/%s for %s: %s.\n",
2956                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
2957                                     v->gv_name, strerror(err));
2958                                 break;
2959 
2960                         default:
2961                                 bad_error(f, err);
2962                         }
2963                 }
2964         }
2965 
2966         for (e = uu_list_first(v->gv_dependencies);
2967             e != NULL;
2968             e = uu_list_next(v->gv_dependencies, e)) {
2969                 r = eval_subgraph(e->ge_vertex, h);
2970                 if (r != 0) {
2971                         assert(r == ECONNABORTED);
2972                         ret = ECONNABORTED;
2973                 }
2974         }
2975 
2976         return (ret);
2977 }
2978 
2979 /*
2980  * Delete the (property group) dependencies of v & create new ones based on
2981  * inst.  If doing so would create a cycle, log a message and put the instance
2982  * into maintenance.  Update GV_INSUBGRAPH flags as necessary.  Returns 0 or
2983  * ECONNABORTED.
2984  */
2985 int
2986 refresh_vertex(graph_vertex_t *v, scf_instance_t *inst)
2987 {
2988         int err;
2989         int *path;
2990         char *fmri;
2991         int r;
2992         scf_handle_t *h = scf_instance_handle(inst);
2993         uu_list_t *old_deps;
2994         int ret = 0;
2995         graph_edge_t *e;
2996         graph_vertex_t *vv;
2997 
2998         assert(MUTEX_HELD(&dgraph_lock));
2999         assert(v->gv_type == GVT_INST);
3000 
3001         log_framework(LOG_DEBUG, "Graph engine: Refreshing %s.\n", v->gv_name);
3002 
3003         if (milestone > MILESTONE_NONE) {
3004                 /*
3005                  * In case some of v's dependencies are being deleted we must
3006                  * make a list of them now for GV_INSUBGRAPH-flag evaluation
3007                  * after the new dependencies are in place.
3008                  */
3009                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
3010 
3011                 err = uu_list_walk(v->gv_dependencies,
3012                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
3013                 assert(err == 0);
3014         }
3015 
3016         delete_instance_dependencies(v, B_FALSE);
3017 
3018         err = set_dependencies(v, inst, &path);
3019         switch (err) {
3020         case 0:
3021                 break;
3022 
3023         case ECONNABORTED:
3024                 ret = err;
3025                 goto out;
3026 
3027         case EINVAL:
3028         case ELOOP:
3029                 r = libscf_instance_get_fmri(inst, &fmri);
3030                 switch (r) {
3031                 case 0:
3032                         break;
3033 
3034                 case ECONNABORTED:
3035                         ret = ECONNABORTED;
3036                         goto out;
3037 
3038                 case ECANCELED:
3039                         ret = 0;
3040                         goto out;
3041 
3042                 default:
3043                         bad_error("libscf_instance_get_fmri", r);
3044                 }
3045 
3046                 if (err == EINVAL) {
3047                         log_error(LOG_ERR, "Transitioning %s "
3048                             "to maintenance due to misconfiguration.\n",
3049                             fmri ? fmri : "?");
3050                         vertex_send_event(v,
3051                             RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY);
3052                 } else {
3053                         handle_cycle(fmri, path);
3054                         vertex_send_event(v,
3055                             RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE);
3056                 }
3057                 startd_free(fmri, max_scf_fmri_size);
3058                 ret = 0;
3059                 goto out;
3060 
3061         default:
3062                 bad_error("set_dependencies", err);
3063         }
3064 
3065         if (milestone > MILESTONE_NONE) {
3066                 boolean_t aborted = B_FALSE;
3067 
3068                 for (e = uu_list_first(old_deps);
3069                     e != NULL;
3070                     e = uu_list_next(old_deps, e)) {
3071                         vv = e->ge_vertex;
3072 
3073                         if (vertex_unref(vv) == VERTEX_INUSE &&
3074                             eval_subgraph(vv, h) == ECONNABORTED)
3075                                 aborted = B_TRUE;
3076                 }
3077 
3078                 for (e = uu_list_first(v->gv_dependencies);
3079                     e != NULL;
3080                     e = uu_list_next(v->gv_dependencies, e)) {
3081                         if (eval_subgraph(e->ge_vertex, h) ==
3082                             ECONNABORTED)
3083                                 aborted = B_TRUE;
3084                 }
3085 
3086                 if (aborted) {
3087                         ret = ECONNABORTED;
3088                         goto out;
3089                 }
3090         }
3091 
3092         graph_start_if_satisfied(v);
3093 
3094         ret = 0;
3095 
3096 out:
3097         if (milestone > MILESTONE_NONE) {
3098                 void *cookie = NULL;
3099 
3100                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL)
3101                         startd_free(e, sizeof (*e));
3102 
3103                 uu_list_destroy(old_deps);
3104         }
3105 
3106         return (ret);
3107 }
3108 
3109 /*
3110  * Set up v according to inst.  That is, make sure it depends on its
3111  * restarter and set up its dependencies.  Send the ADD_INSTANCE command to
3112  * the restarter, and send ENABLE or DISABLE as appropriate.
3113  *
3114  * Returns 0 on success, ECONNABORTED on repository disconnection, or
3115  * ECANCELED if inst is deleted.
3116  */
3117 static int
3118 configure_vertex(graph_vertex_t *v, scf_instance_t *inst)
3119 {
3120         scf_handle_t *h;
3121         scf_propertygroup_t *pg;
3122         scf_snapshot_t *snap;
3123         char *restarter_fmri = startd_alloc(max_scf_value_size);
3124         int enabled, enabled_ovr;
3125         int err;
3126         int *path;
3127         int deathrow;
3128         int32_t tset;
3129 
3130         restarter_fmri[0] = '\0';
3131 
3132         assert(MUTEX_HELD(&dgraph_lock));
3133         assert(v->gv_type == GVT_INST);
3134         assert((v->gv_flags & GV_CONFIGURED) == 0);
3135 
3136         /* GV_INSUBGRAPH should already be set properly. */
3137         assert(should_be_in_subgraph(v) ==
3138             ((v->gv_flags & GV_INSUBGRAPH) != 0));
3139 
3140         /*
3141          * If the instance fmri is in the deathrow list then set the
3142          * GV_DEATHROW flag on the vertex and create and set to true the
3143          * SCF_PROPERTY_DEATHROW boolean property in the non-persistent
3144          * repository for this instance fmri.
3145          */
3146         if ((v->gv_flags & GV_DEATHROW) ||
3147             (is_fmri_in_deathrow(v->gv_name) == B_TRUE)) {
3148                 if ((v->gv_flags & GV_DEATHROW) == 0) {
3149                         /*
3150                          * Set flag GV_DEATHROW, create and set to true
3151                          * the SCF_PROPERTY_DEATHROW property in the
3152                          * non-persistent repository for this instance fmri.
3153                          */
3154                         v->gv_flags |= GV_DEATHROW;
3155 
3156                         switch (err = libscf_set_deathrow(inst, 1)) {
3157                         case 0:
3158                                 break;
3159 
3160                         case ECONNABORTED:
3161                         case ECANCELED:
3162                                 startd_free(restarter_fmri, max_scf_value_size);
3163                                 return (err);
3164 
3165                         case EROFS:
3166                                 log_error(LOG_WARNING, "Could not set %s/%s "
3167                                     "for deathrow %s: %s.\n",
3168                                     SCF_PG_DEATHROW, SCF_PROPERTY_DEATHROW,
3169                                     v->gv_name, strerror(err));
3170                                 break;
3171 
3172                         case EPERM:
3173                                 uu_die("Permission denied.\n");
3174                                 /* NOTREACHED */
3175 
3176                         default:
3177                                 bad_error("libscf_set_deathrow", err);
3178                         }
3179                         log_framework(LOG_DEBUG, "Deathrow, graph set %s.\n",
3180                             v->gv_name);
3181                 }
3182                 startd_free(restarter_fmri, max_scf_value_size);
3183                 return (0);
3184         }
3185 
3186         h = scf_instance_handle(inst);
3187 
3188         /*
3189          * Using a temporary deathrow boolean property, set through
3190          * libscf_set_deathrow(), only for fmris on deathrow, is necessary
3191          * because deathrow_fini() may already have been called, and in case
3192          * of a refresh, GV_DEATHROW may need to be set again.
3193          * libscf_get_deathrow() sets deathrow to 1 only if this instance
3194          * has a temporary boolean property named 'deathrow' valued true
3195          * in a property group 'deathrow', -1 or 0 in all other cases.
3196          */
3197         err = libscf_get_deathrow(h, inst, &deathrow);
3198         switch (err) {
3199         case 0:
3200                 break;
3201 
3202         case ECONNABORTED:
3203         case ECANCELED:
3204                 startd_free(restarter_fmri, max_scf_value_size);
3205                 return (err);
3206 
3207         default:
3208                 bad_error("libscf_get_deathrow", err);
3209         }
3210 
3211         if (deathrow == 1) {
3212                 v->gv_flags |= GV_DEATHROW;
3213                 startd_free(restarter_fmri, max_scf_value_size);
3214                 return (0);
3215         }
3216 
3217         log_framework(LOG_DEBUG, "Graph adding %s.\n", v->gv_name);
3218 
3219         /*
3220          * If the instance does not have a restarter property group,
3221          * initialize its state to uninitialized/none, in case the restarter
3222          * is not enabled.
3223          */
3224         pg = safe_scf_pg_create(h);
3225 
3226         if (scf_instance_get_pg(inst, SCF_PG_RESTARTER, pg) != 0) {
3227                 instance_data_t idata;
3228                 uint_t count = 0, msecs = ALLOC_DELAY;
3229 
3230                 switch (scf_error()) {
3231                 case SCF_ERROR_NOT_FOUND:
3232                         break;
3233 
3234                 case SCF_ERROR_CONNECTION_BROKEN:
3235                 default:
3236                         scf_pg_destroy(pg);
3237                         startd_free(restarter_fmri, max_scf_value_size);
3238                         return (ECONNABORTED);
3239 
3240                 case SCF_ERROR_DELETED:
3241                         scf_pg_destroy(pg);
3242                         startd_free(restarter_fmri, max_scf_value_size);
3243                         return (ECANCELED);
3244 
3245                 case SCF_ERROR_NOT_SET:
3246                         bad_error("scf_instance_get_pg", scf_error());
3247                 }
3248 
3249                 switch (err = libscf_instance_get_fmri(inst,
3250                     (char **)&idata.i_fmri)) {
3251                 case 0:
3252                         break;
3253 
3254                 case ECONNABORTED:
3255                 case ECANCELED:
3256                         scf_pg_destroy(pg);
3257                         startd_free(restarter_fmri, max_scf_value_size);
3258                         return (err);
3259 
3260                 default:
3261                         bad_error("libscf_instance_get_fmri", err);
3262                 }
3263 
3264                 idata.i_state = RESTARTER_STATE_NONE;
3265                 idata.i_next_state = RESTARTER_STATE_NONE;
3266 
3267 init_state:
3268                 switch (err = _restarter_commit_states(h, &idata,
3269                     RESTARTER_STATE_UNINIT, RESTARTER_STATE_NONE,
3270                     restarter_get_str_short(restarter_str_insert_in_graph))) {
3271                 case 0:
3272                         break;
3273 
3274                 case ENOMEM:
3275                         ++count;
3276                         if (count < ALLOC_RETRY) {
3277                                 (void) poll(NULL, 0, msecs);
3278                                 msecs *= ALLOC_DELAY_MULT;
3279                                 goto init_state;
3280                         }
3281 
3282                         uu_die("Insufficient memory.\n");
3283                         /* NOTREACHED */
3284 
3285                 case ECONNABORTED:
3286                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3287                         scf_pg_destroy(pg);
3288                         startd_free(restarter_fmri, max_scf_value_size);
3289                         return (ECONNABORTED);
3290 
3291                 case ENOENT:
3292                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3293                         scf_pg_destroy(pg);
3294                         startd_free(restarter_fmri, max_scf_value_size);
3295                         return (ECANCELED);
3296 
3297                 case EPERM:
3298                 case EACCES:
3299                 case EROFS:
3300                         log_error(LOG_NOTICE, "Could not initialize state for "
3301                             "%s: %s.\n", idata.i_fmri, strerror(err));
3302                         break;
3303 
3304                 case EINVAL:
3305                 default:
3306                         bad_error("_restarter_commit_states", err);
3307                 }
3308 
3309                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3310         }
3311 
3312         scf_pg_destroy(pg);
3313 
3314         if (milestone != NULL) {
3315                 /*
3316                  * Make sure the enable-override is set properly before we
3317                  * read whether we should be enabled.
3318                  */
3319                 if (milestone == MILESTONE_NONE ||
3320                     !(v->gv_flags & GV_INSUBGRAPH)) {
3321                         /*
3322                          * This might seem unjustified after the milestone
3323                          * transition has completed (non_subgraph_svcs == 0),
3324                          * but it's important because when we boot to
3325                          * a milestone, we set the milestone before populating
3326                          * the graph, and all of the new non-subgraph services
3327                          * need to be disabled here.
3328                          */
3329                         switch (err = libscf_set_enable_ovr(inst, 0)) {
3330                         case 0:
3331                                 break;
3332 
3333                         case ECONNABORTED:
3334                         case ECANCELED:
3335                                 startd_free(restarter_fmri, max_scf_value_size);
3336                                 return (err);
3337 
3338                         case EROFS:
3339                                 log_error(LOG_WARNING,
3340                                     "Could not set %s/%s for %s: %s.\n",
3341                                     SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
3342                                     v->gv_name, strerror(err));
3343                                 break;
3344 
3345                         case EPERM:
3346                                 uu_die("Permission denied.\n");
3347                                 /* NOTREACHED */
3348 
3349                         default:
3350                                 bad_error("libscf_set_enable_ovr", err);
3351                         }
3352                 } else {
3353                         assert(v->gv_flags & GV_INSUBGRAPH);
3354                         switch (err = libscf_delete_enable_ovr(inst)) {
3355                         case 0:
3356                                 break;
3357 
3358                         case ECONNABORTED:
3359                         case ECANCELED:
3360                                 startd_free(restarter_fmri, max_scf_value_size);
3361                                 return (err);
3362 
3363                         case EPERM:
3364                                 uu_die("Permission denied.\n");
3365                                 /* NOTREACHED */
3366 
3367                         default:
3368                                 bad_error("libscf_delete_enable_ovr", err);
3369                         }
3370                 }
3371         }
3372 
3373         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
3374             &enabled_ovr, &restarter_fmri);
3375         switch (err) {
3376         case 0:
3377                 break;
3378 
3379         case ECONNABORTED:
3380         case ECANCELED:
3381                 startd_free(restarter_fmri, max_scf_value_size);
3382                 return (err);
3383 
3384         case ENOENT:
3385                 log_framework(LOG_DEBUG,
3386                     "Ignoring %s because it has no general property group.\n",
3387                     v->gv_name);
3388                 startd_free(restarter_fmri, max_scf_value_size);
3389                 return (0);
3390 
3391         default:
3392                 bad_error("libscf_get_basic_instance_data", err);
3393         }
3394 
3395         if ((tset = libscf_get_stn_tset(inst)) == -1) {
3396                 log_framework(LOG_WARNING,
3397                     "Failed to get notification parameters for %s: %s\n",
3398                     v->gv_name, scf_strerror(scf_error()));
3399                 v->gv_stn_tset = 0;
3400         } else {
3401                 v->gv_stn_tset = tset;
3402         }
3403         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
3404                 stn_global = v->gv_stn_tset;
3405 
3406         if (enabled == -1) {
3407                 startd_free(restarter_fmri, max_scf_value_size);
3408                 return (0);
3409         }
3410 
3411         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
3412             (enabled ? GV_ENBLD_NOOVR : 0);
3413 
3414         if (enabled_ovr != -1)
3415                 enabled = enabled_ovr;
3416 
3417         v->gv_state = RESTARTER_STATE_UNINIT;
3418 
3419         snap = libscf_get_or_make_running_snapshot(inst, v->gv_name, B_TRUE);
3420         scf_snapshot_destroy(snap);
3421 
3422         /* Set up the restarter. (Sends _ADD_INSTANCE on success.) */
3423         err = graph_change_restarter(v, restarter_fmri, h, &path);
3424         if (err != 0) {
3425                 instance_data_t idata;
3426                 uint_t count = 0, msecs = ALLOC_DELAY;
3427                 restarter_str_t reason;
3428 
3429                 if (err == ECONNABORTED) {
3430                         startd_free(restarter_fmri, max_scf_value_size);
3431                         return (err);
3432                 }
3433 
3434                 assert(err == EINVAL || err == ELOOP);
3435 
3436                 if (err == EINVAL) {
3437                         log_framework(LOG_ERR, emsg_invalid_restarter,
3438                             v->gv_name, restarter_fmri);
3439                         reason = restarter_str_invalid_restarter;
3440                 } else {
3441                         handle_cycle(v->gv_name, path);
3442                         reason = restarter_str_dependency_cycle;
3443                 }
3444 
3445                 startd_free(restarter_fmri, max_scf_value_size);
3446 
3447                 /*
3448                  * We didn't register the instance with the restarter, so we
3449                  * must set maintenance mode ourselves.
3450                  */
3451                 err = libscf_instance_get_fmri(inst, (char **)&idata.i_fmri);
3452                 if (err != 0) {
3453                         assert(err == ECONNABORTED || err == ECANCELED);
3454                         return (err);
3455                 }
3456 
3457                 idata.i_state = RESTARTER_STATE_NONE;
3458                 idata.i_next_state = RESTARTER_STATE_NONE;
3459 
3460 set_maint:
3461                 switch (err = _restarter_commit_states(h, &idata,
3462                     RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE,
3463                     restarter_get_str_short(reason))) {
3464                 case 0:
3465                         break;
3466 
3467                 case ENOMEM:
3468                         ++count;
3469                         if (count < ALLOC_RETRY) {
3470                                 (void) poll(NULL, 0, msecs);
3471                                 msecs *= ALLOC_DELAY_MULT;
3472                                 goto set_maint;
3473                         }
3474 
3475                         uu_die("Insufficient memory.\n");
3476                         /* NOTREACHED */
3477 
3478                 case ECONNABORTED:
3479                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3480                         return (ECONNABORTED);
3481 
3482                 case ENOENT:
3483                         startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3484                         return (ECANCELED);
3485 
3486                 case EPERM:
3487                 case EACCES:
3488                 case EROFS:
3489                         log_error(LOG_NOTICE, "Could not initialize state for "
3490                             "%s: %s.\n", idata.i_fmri, strerror(err));
3491                         break;
3492 
3493                 case EINVAL:
3494                 default:
3495                         bad_error("_restarter_commit_states", err);
3496                 }
3497 
3498                 startd_free((void *)idata.i_fmri, max_scf_fmri_size);
3499 
3500                 v->gv_state = RESTARTER_STATE_MAINT;
3501 
3502                 goto out;
3503         }
3504         startd_free(restarter_fmri, max_scf_value_size);
3505 
3506         /* Add all the other dependencies. */
3507         err = refresh_vertex(v, inst);
3508         if (err != 0) {
3509                 assert(err == ECONNABORTED);
3510                 return (err);
3511         }
3512 
3513 out:
3514         v->gv_flags |= GV_CONFIGURED;
3515 
3516         graph_enable_by_vertex(v, enabled, 0);
3517 
3518         return (0);
3519 }
3520 
3521 
3522 static void
3523 kill_user_procs(void)
3524 {
3525         (void) fputs("svc.startd: Killing user processes.\n", stdout);
3526 
3527         /*
3528          * Despite its name, killall's role is to get select user processes--
3529          * basically those representing terminal-based logins-- to die.  Victims
3530          * are located by killall in the utmp database.  Since these are most
3531          * often shell based logins, and many shells mask SIGTERM (but are
3532          * responsive to SIGHUP) we first HUP and then shortly thereafter
3533          * kill -9.
3534          */
3535         (void) fork_with_timeout("/usr/sbin/killall HUP", 1, 5);
3536         (void) fork_with_timeout("/usr/sbin/killall KILL", 1, 5);
3537 
3538         /*
3539          * Note the selection of user id's 0, 1 and 15, subsequently
3540          * inverted by -v.  15 is reserved for dladmd.  Yes, this is a
3541          * kludge-- a better policy is needed.
3542          *
3543          * Note that fork_with_timeout will only wait out the 1 second
3544          * "grace time" if pkill actually returns 0.  So if there are
3545          * no matches, this will run to completion much more quickly.
3546          */
3547         (void) fork_with_timeout("/usr/bin/pkill -TERM -v -u 0,1,15", 1, 5);
3548         (void) fork_with_timeout("/usr/bin/pkill -KILL -v -u 0,1,15", 1, 5);
3549 }
3550 
3551 static void
3552 do_uadmin(void)
3553 {
3554         const char * const resetting = "/etc/svc/volatile/resetting";
3555         int fd;
3556         struct statvfs vfs;
3557         time_t now;
3558         struct tm nowtm;
3559         char down_buf[256], time_buf[256];
3560         uintptr_t mdep;
3561 #if defined(__x86)
3562         char *fbarg = NULL;
3563 #endif  /* __x86 */
3564 
3565         mdep = NULL;
3566         fd = creat(resetting, 0777);
3567         if (fd >= 0)
3568                 startd_close(fd);
3569         else
3570                 uu_warn("Could not create \"%s\"", resetting);
3571 
3572         /* Kill dhcpagent if we're not using nfs for root */
3573         if ((statvfs("/", &vfs) == 0) &&
3574             (strncmp(vfs.f_basetype, "nfs", sizeof ("nfs") - 1) != 0))
3575                 fork_with_timeout("/usr/bin/pkill -x -u 0 dhcpagent", 0, 5);
3576 
3577         /*
3578          * Call sync(2) now, before we kill off user processes.  This takes
3579          * advantage of the several seconds of pause we have before the
3580          * killalls are done.  Time we can make good use of to get pages
3581          * moving out to disk.
3582          *
3583          * Inside non-global zones, we don't bother, and it's better not to
3584          * anyway, since sync(2) can have system-wide impact.
3585          */
3586         if (getzoneid() == 0)
3587                 sync();
3588 
3589         kill_user_procs();
3590 
3591         /*
3592          * Note that this must come after the killing of user procs, since
3593          * killall relies on utmpx, and this command affects the contents of
3594          * said file.
3595          */
3596         if (access("/usr/lib/acct/closewtmp", X_OK) == 0)
3597                 fork_with_timeout("/usr/lib/acct/closewtmp", 0, 5);
3598 
3599         /*
3600          * For patches which may be installed as the system is shutting
3601          * down, we need to ensure, one more time, that the boot archive
3602          * really is up to date.
3603          */
3604         if (getzoneid() == 0 && access("/usr/sbin/bootadm", X_OK) == 0)
3605                 fork_with_timeout("/usr/sbin/bootadm -ea update_all", 0, 3600);
3606 
3607         /*
3608          * Right now, fast reboot is supported only on i386.
3609          * scf_is_fastboot_default() should take care of it.
3610          * If somehow we got there on unsupported platform -
3611          * print warning and fall back to regular reboot.
3612          */
3613         if (halting == AD_FASTREBOOT) {
3614 #if defined(__x86)
3615                 if (be_get_boot_args(&fbarg, BE_ENTRY_DEFAULT) == 0) {
3616                         mdep = (uintptr_t)fbarg;
3617                 } else {
3618                         /*
3619                          * Failed to read BE info, fall back to normal reboot
3620                          */
3621                         halting = AD_BOOT;
3622                         uu_warn("Failed to get fast reboot arguments.\n"
3623                             "Falling back to regular reboot.\n");
3624                 }
3625 #else   /* __x86 */
3626                 halting = AD_BOOT;
3627                 uu_warn("Fast reboot configured, but not supported by "
3628                     "this ISA\n");
3629 #endif  /* __x86 */
3630         }
3631 
3632         fork_with_timeout("/sbin/umountall -l", 0, 5);
3633         fork_with_timeout("/sbin/umount /tmp /var/adm /var/run /var "
3634             ">/dev/null 2>&1", 0, 5);
3635 
3636         /*
3637          * Try to get to consistency for whatever UFS filesystems are left.
3638          * This is pretty expensive, so we save it for the end in the hopes of
3639          * minimizing what it must do.  The other option would be to start in
3640          * parallel with the killall's, but lockfs tends to throw out much more
3641          * than is needed, and so subsequent commands (like umountall) take a
3642          * long time to get going again.
3643          *
3644          * Inside of zones, we don't bother, since we're not about to terminate
3645          * the whole OS instance.
3646          *
3647          * On systems using only ZFS, this call to lockfs -fa is a no-op.
3648          */
3649         if (getzoneid() == 0) {
3650                 if (access("/usr/sbin/lockfs", X_OK) == 0)
3651                         fork_with_timeout("/usr/sbin/lockfs -fa", 0, 30);
3652 
3653                 sync(); /* once more, with feeling */
3654         }
3655 
3656         fork_with_timeout("/sbin/umount /usr >/dev/null 2>&1", 0, 5);
3657 
3658         /*
3659          * Construct and emit the last words from userland:
3660          * "<timestamp> The system is down.  Shutdown took <N> seconds."
3661          *
3662          * Normally we'd use syslog, but with /var and other things
3663          * potentially gone, try to minimize the external dependencies.
3664          */
3665         now = time(NULL);
3666         (void) localtime_r(&now, &nowtm);
3667 
3668         if (strftime(down_buf, sizeof (down_buf),
3669             "%b %e %T The system is down.", &nowtm) == 0) {
3670                 (void) strlcpy(down_buf, "The system is down.",
3671                     sizeof (down_buf));
3672         }
3673 
3674         if (halting_time != 0 && halting_time <= now) {
3675                 (void) snprintf(time_buf, sizeof (time_buf),
3676                     "  Shutdown took %lu seconds.", now - halting_time);
3677         } else {
3678                 time_buf[0] = '\0';
3679         }
3680         (void) printf("%s%s\n", down_buf, time_buf);
3681 
3682         (void) uadmin(A_SHUTDOWN, halting, mdep);
3683         uu_warn("uadmin() failed");
3684 
3685 #if defined(__x86)
3686         if (halting == AD_FASTREBOOT)
3687                 free(fbarg);
3688 #endif  /* __x86 */
3689 
3690         if (remove(resetting) != 0 && errno != ENOENT)
3691                 uu_warn("Could not remove \"%s\"", resetting);
3692 }
3693 
3694 /*
3695  * If any of the up_svcs[] are online or satisfiable, return true.  If they are
3696  * all missing, disabled, in maintenance, or unsatisfiable, return false.
3697  */
3698 boolean_t
3699 can_come_up(void)
3700 {
3701         int i;
3702 
3703         assert(MUTEX_HELD(&dgraph_lock));
3704 
3705         /*
3706          * If we are booting to single user (boot -s),
3707          * SCF_MILESTONE_SINGLE_USER is needed to come up because startd
3708          * spawns sulogin after single-user is online (see specials.c).
3709          */
3710         i = (booting_to_single_user ? 0 : 1);
3711 
3712         for (; up_svcs[i] != NULL; ++i) {
3713                 if (up_svcs_p[i] == NULL) {
3714                         up_svcs_p[i] = vertex_get_by_name(up_svcs[i]);
3715 
3716                         if (up_svcs_p[i] == NULL)
3717                                 continue;
3718                 }
3719 
3720                 /*
3721                  * Ignore unconfigured services (the ones that have been
3722                  * mentioned in a dependency from other services, but do
3723                  * not exist in the repository).  Services which exist
3724                  * in the repository but don't have general/enabled
3725                  * property will be also ignored.
3726                  */
3727                 if (!(up_svcs_p[i]->gv_flags & GV_CONFIGURED))
3728                         continue;
3729 
3730                 switch (up_svcs_p[i]->gv_state) {
3731                 case RESTARTER_STATE_ONLINE:
3732                 case RESTARTER_STATE_DEGRADED:
3733                         /*
3734                          * Deactivate verbose boot once a login service has been
3735                          * reached.
3736                          */
3737                         st->st_log_login_reached = 1;
3738                         /*FALLTHROUGH*/
3739                 case RESTARTER_STATE_UNINIT:
3740                         return (B_TRUE);
3741 
3742                 case RESTARTER_STATE_OFFLINE:
3743                         if (instance_satisfied(up_svcs_p[i], B_TRUE) != -1)
3744                                 return (B_TRUE);
3745                         log_framework(LOG_DEBUG,
3746                             "can_come_up(): %s is unsatisfiable.\n",
3747                             up_svcs_p[i]->gv_name);
3748                         continue;
3749 
3750                 case RESTARTER_STATE_DISABLED:
3751                 case RESTARTER_STATE_MAINT:
3752                         log_framework(LOG_DEBUG,
3753                             "can_come_up(): %s is in state %s.\n",
3754                             up_svcs_p[i]->gv_name,
3755                             instance_state_str[up_svcs_p[i]->gv_state]);
3756                         continue;
3757 
3758                 default:
3759 #ifndef NDEBUG
3760                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
3761                             __FILE__, __LINE__, up_svcs_p[i]->gv_state);
3762 #endif
3763                         abort();
3764                 }
3765         }
3766 
3767         /*
3768          * In the seed repository, console-login is unsatisfiable because
3769          * services are missing.  To behave correctly in that case we don't want
3770          * to return false until manifest-import is online.
3771          */
3772 
3773         if (manifest_import_p == NULL) {
3774                 manifest_import_p = vertex_get_by_name(manifest_import);
3775 
3776                 if (manifest_import_p == NULL)
3777                         return (B_FALSE);
3778         }
3779 
3780         switch (manifest_import_p->gv_state) {
3781         case RESTARTER_STATE_ONLINE:
3782         case RESTARTER_STATE_DEGRADED:
3783         case RESTARTER_STATE_DISABLED:
3784         case RESTARTER_STATE_MAINT:
3785                 break;
3786 
3787         case RESTARTER_STATE_OFFLINE:
3788                 if (instance_satisfied(manifest_import_p, B_TRUE) == -1)
3789                         break;
3790                 /* FALLTHROUGH */
3791 
3792         case RESTARTER_STATE_UNINIT:
3793                 return (B_TRUE);
3794         }
3795 
3796         return (B_FALSE);
3797 }
3798 
3799 /*
3800  * Runs sulogin.  Returns
3801  *   0 - success
3802  *   EALREADY - sulogin is already running
3803  *   EBUSY - console-login is running
3804  */
3805 static int
3806 run_sulogin(const char *msg)
3807 {
3808         graph_vertex_t *v;
3809 
3810         assert(MUTEX_HELD(&dgraph_lock));
3811 
3812         if (sulogin_running)
3813                 return (EALREADY);
3814 
3815         v = vertex_get_by_name(console_login_fmri);
3816         if (v != NULL && inst_running(v))
3817                 return (EBUSY);
3818 
3819         sulogin_running = B_TRUE;
3820 
3821         MUTEX_UNLOCK(&dgraph_lock);
3822 
3823         fork_sulogin(B_FALSE, msg);
3824 
3825         MUTEX_LOCK(&dgraph_lock);
3826 
3827         sulogin_running = B_FALSE;
3828 
3829         if (console_login_ready) {
3830                 v = vertex_get_by_name(console_login_fmri);
3831 
3832                 if (v != NULL && v->gv_state == RESTARTER_STATE_OFFLINE) {
3833                         if (v->gv_start_f == NULL)
3834                                 vertex_send_event(v,
3835                                     RESTARTER_EVENT_TYPE_START);
3836                         else
3837                                 v->gv_start_f(v);
3838                 }
3839 
3840                 console_login_ready = B_FALSE;
3841         }
3842 
3843         return (0);
3844 }
3845 
3846 /*
3847  * The sulogin thread runs sulogin while can_come_up() is false.  run_sulogin()
3848  * keeps sulogin from stepping on console-login's toes.
3849  */
3850 /* ARGSUSED */
3851 static void *
3852 sulogin_thread(void *unused)
3853 {
3854         MUTEX_LOCK(&dgraph_lock);
3855 
3856         assert(sulogin_thread_running);
3857 
3858         do {
3859                 (void) run_sulogin("Console login service(s) cannot run\n");
3860         } while (!can_come_up());
3861 
3862         sulogin_thread_running = B_FALSE;
3863         MUTEX_UNLOCK(&dgraph_lock);
3864 
3865         return (NULL);
3866 }
3867 
3868 /* ARGSUSED */
3869 void *
3870 single_user_thread(void *unused)
3871 {
3872         uint_t left;
3873         scf_handle_t *h;
3874         scf_instance_t *inst;
3875         scf_property_t *prop;
3876         scf_value_t *val;
3877         const char *msg;
3878         char *buf;
3879         int r;
3880 
3881         MUTEX_LOCK(&single_user_thread_lock);
3882         single_user_thread_count++;
3883 
3884         if (!booting_to_single_user)
3885                 kill_user_procs();
3886 
3887         if (go_single_user_mode || booting_to_single_user) {
3888                 msg = "SINGLE USER MODE\n";
3889         } else {
3890                 assert(go_to_level1);
3891 
3892                 fork_rc_script('1', "start", B_TRUE);
3893 
3894                 uu_warn("The system is ready for administration.\n");
3895 
3896                 msg = "";
3897         }
3898 
3899         MUTEX_UNLOCK(&single_user_thread_lock);
3900 
3901         for (;;) {
3902                 MUTEX_LOCK(&dgraph_lock);
3903                 r = run_sulogin(msg);
3904                 MUTEX_UNLOCK(&dgraph_lock);
3905                 if (r == 0)
3906                         break;
3907 
3908                 assert(r == EALREADY || r == EBUSY);
3909 
3910                 left = 3;
3911                 while (left > 0)
3912                         left = sleep(left);
3913         }
3914 
3915         MUTEX_LOCK(&single_user_thread_lock);
3916 
3917         /*
3918          * If another single user thread has started, let it finish changing
3919          * the run level.
3920          */
3921         if (single_user_thread_count > 1) {
3922                 single_user_thread_count--;
3923                 MUTEX_UNLOCK(&single_user_thread_lock);
3924                 return (NULL);
3925         }
3926 
3927         h = libscf_handle_create_bound_loop();
3928         inst = scf_instance_create(h);
3929         prop = safe_scf_property_create(h);
3930         val = safe_scf_value_create(h);
3931         buf = startd_alloc(max_scf_fmri_size);
3932 
3933 lookup:
3934         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
3935             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
3936                 switch (scf_error()) {
3937                 case SCF_ERROR_NOT_FOUND:
3938                         r = libscf_create_self(h);
3939                         if (r == 0)
3940                                 goto lookup;
3941                         assert(r == ECONNABORTED);
3942                         /* FALLTHROUGH */
3943 
3944                 case SCF_ERROR_CONNECTION_BROKEN:
3945                         libscf_handle_rebind(h);
3946                         goto lookup;
3947 
3948                 case SCF_ERROR_INVALID_ARGUMENT:
3949                 case SCF_ERROR_CONSTRAINT_VIOLATED:
3950                 case SCF_ERROR_NOT_BOUND:
3951                 case SCF_ERROR_HANDLE_MISMATCH:
3952                 default:
3953                         bad_error("scf_handle_decode_fmri", scf_error());
3954                 }
3955         }
3956 
3957         MUTEX_LOCK(&dgraph_lock);
3958 
3959         r = scf_instance_delete_prop(inst, SCF_PG_OPTIONS_OVR,
3960             SCF_PROPERTY_MILESTONE);
3961         switch (r) {
3962         case 0:
3963         case ECANCELED:
3964                 break;
3965 
3966         case ECONNABORTED:
3967                 MUTEX_UNLOCK(&dgraph_lock);
3968                 libscf_handle_rebind(h);
3969                 goto lookup;
3970 
3971         case EPERM:
3972         case EACCES:
3973         case EROFS:
3974                 log_error(LOG_WARNING, "Could not clear temporary milestone: "
3975                     "%s.\n", strerror(r));
3976                 break;
3977 
3978         default:
3979                 bad_error("scf_instance_delete_prop", r);
3980         }
3981 
3982         MUTEX_UNLOCK(&dgraph_lock);
3983 
3984         r = libscf_get_milestone(inst, prop, val, buf, max_scf_fmri_size);
3985         switch (r) {
3986         case ECANCELED:
3987         case ENOENT:
3988         case EINVAL:
3989                 (void) strcpy(buf, "all");
3990                 /* FALLTHROUGH */
3991 
3992         case 0:
3993                 uu_warn("Returning to milestone %s.\n", buf);
3994                 break;
3995 
3996         case ECONNABORTED:
3997                 libscf_handle_rebind(h);
3998                 goto lookup;
3999 
4000         default:
4001                 bad_error("libscf_get_milestone", r);
4002         }
4003 
4004         r = dgraph_set_milestone(buf, h, B_FALSE);
4005         switch (r) {
4006         case 0:
4007         case ECONNRESET:
4008         case EALREADY:
4009         case EINVAL:
4010         case ENOENT:
4011                 break;
4012 
4013         default:
4014                 bad_error("dgraph_set_milestone", r);
4015         }
4016 
4017         /*
4018          * See graph_runlevel_changed().
4019          */
4020         MUTEX_LOCK(&dgraph_lock);
4021         utmpx_set_runlevel(target_milestone_as_runlevel(), 'S', B_TRUE);
4022         MUTEX_UNLOCK(&dgraph_lock);
4023 
4024         startd_free(buf, max_scf_fmri_size);
4025         scf_value_destroy(val);
4026         scf_property_destroy(prop);
4027         scf_instance_destroy(inst);
4028         scf_handle_destroy(h);
4029 
4030         /*
4031          * We'll give ourselves 3 seconds to respond to all of the enablings
4032          * that setting the milestone should have created before checking
4033          * whether to run sulogin.
4034          */
4035         left = 3;
4036         while (left > 0)
4037                 left = sleep(left);
4038 
4039         MUTEX_LOCK(&dgraph_lock);
4040         /*
4041          * Clearing these variables will allow the sulogin thread to run.  We
4042          * check here in case there aren't any more state updates anytime soon.
4043          */
4044         go_to_level1 = go_single_user_mode = booting_to_single_user = B_FALSE;
4045         if (!sulogin_thread_running && !can_come_up()) {
4046                 (void) startd_thread_create(sulogin_thread, NULL);
4047                 sulogin_thread_running = B_TRUE;
4048         }
4049         MUTEX_UNLOCK(&dgraph_lock);
4050         single_user_thread_count--;
4051         MUTEX_UNLOCK(&single_user_thread_lock);
4052         return (NULL);
4053 }
4054 
4055 
4056 /*
4057  * Dependency graph operations API.  These are handle-independent thread-safe
4058  * graph manipulation functions which are the entry points for the event
4059  * threads below.
4060  */
4061 
4062 /*
4063  * If a configured vertex exists for inst_fmri, return EEXIST.  If no vertex
4064  * exists for inst_fmri, add one.  Then fetch the restarter from inst, make
4065  * this vertex dependent on it, and send _ADD_INSTANCE to the restarter.
4066  * Fetch whether the instance should be enabled from inst and send _ENABLE or
4067  * _DISABLE as appropriate.  Finally rummage through inst's dependency
4068  * property groups and add vertices and edges as appropriate.  If anything
4069  * goes wrong after sending _ADD_INSTANCE, send _ADMIN_MAINT_ON to put the
4070  * instance in maintenance.  Don't send _START or _STOP until we get a state
4071  * update in case we're being restarted and the service is already running.
4072  *
4073  * To support booting to a milestone, we must also make sure all dependencies
4074  * encountered are configured, if they exist in the repository.
4075  *
4076  * Returns 0 on success, ECONNABORTED on repository disconnection, EINVAL if
4077  * inst_fmri is an invalid (or not canonical) FMRI, ECANCELED if inst is
4078  * deleted, or EEXIST if a configured vertex for inst_fmri already exists.
4079  */
4080 int
4081 dgraph_add_instance(const char *inst_fmri, scf_instance_t *inst,
4082     boolean_t lock_graph)
4083 {
4084         graph_vertex_t *v;
4085         int err;
4086 
4087         if (strcmp(inst_fmri, SCF_SERVICE_STARTD) == 0)
4088                 return (0);
4089 
4090         /* Check for a vertex for inst_fmri. */
4091         if (lock_graph) {
4092                 MUTEX_LOCK(&dgraph_lock);
4093         } else {
4094                 assert(MUTEX_HELD(&dgraph_lock));
4095         }
4096 
4097         v = vertex_get_by_name(inst_fmri);
4098 
4099         if (v != NULL) {
4100                 assert(v->gv_type == GVT_INST);
4101 
4102                 if (v->gv_flags & GV_CONFIGURED) {
4103                         if (lock_graph)
4104                                 MUTEX_UNLOCK(&dgraph_lock);
4105                         return (EEXIST);
4106                 }
4107         } else {
4108                 /* Add the vertex. */
4109                 err = graph_insert_vertex_unconfigured(inst_fmri, GVT_INST, 0,
4110                     RERR_NONE, &v);
4111                 if (err != 0) {
4112                         assert(err == EINVAL);
4113                         if (lock_graph)
4114                                 MUTEX_UNLOCK(&dgraph_lock);
4115                         return (EINVAL);
4116                 }
4117         }
4118 
4119         err = configure_vertex(v, inst);
4120 
4121         if (lock_graph)
4122                 MUTEX_UNLOCK(&dgraph_lock);
4123 
4124         return (err);
4125 }
4126 
4127 /*
4128  * Locate the vertex for this property group's instance.  If it doesn't exist
4129  * or is unconfigured, call dgraph_add_instance() & return.  Otherwise fetch
4130  * the restarter for the instance, and if it has changed, send
4131  * _REMOVE_INSTANCE to the old restarter, remove the dependency, make sure the
4132  * new restarter has a vertex, add a new dependency, and send _ADD_INSTANCE to
4133  * the new restarter.  Then fetch whether the instance should be enabled, and
4134  * if it is different from what we had, or if we changed the restarter, send
4135  * the appropriate _ENABLE or _DISABLE command.
4136  *
4137  * Returns 0 on success, ENOTSUP if the pg's parent is not an instance,
4138  * ECONNABORTED on repository disconnection, ECANCELED if the instance is
4139  * deleted, or -1 if the instance's general property group is deleted or if
4140  * its enabled property is misconfigured.
4141  */
4142 static int
4143 dgraph_update_general(scf_propertygroup_t *pg)
4144 {
4145         scf_handle_t *h;
4146         scf_instance_t *inst;
4147         char *fmri;
4148         char *restarter_fmri;
4149         graph_vertex_t *v;
4150         int err;
4151         int enabled, enabled_ovr;
4152         int oldflags;
4153 
4154         /* Find the vertex for this service */
4155         h = scf_pg_handle(pg);
4156 
4157         inst = safe_scf_instance_create(h);
4158 
4159         if (scf_pg_get_parent_instance(pg, inst) != 0) {
4160                 switch (scf_error()) {
4161                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4162                         return (ENOTSUP);
4163 
4164                 case SCF_ERROR_CONNECTION_BROKEN:
4165                 default:
4166                         return (ECONNABORTED);
4167 
4168                 case SCF_ERROR_DELETED:
4169                         return (0);
4170 
4171                 case SCF_ERROR_NOT_SET:
4172                         bad_error("scf_pg_get_parent_instance", scf_error());
4173                 }
4174         }
4175 
4176         err = libscf_instance_get_fmri(inst, &fmri);
4177         switch (err) {
4178         case 0:
4179                 break;
4180 
4181         case ECONNABORTED:
4182                 scf_instance_destroy(inst);
4183                 return (ECONNABORTED);
4184 
4185         case ECANCELED:
4186                 scf_instance_destroy(inst);
4187                 return (0);
4188 
4189         default:
4190                 bad_error("libscf_instance_get_fmri", err);
4191         }
4192 
4193         log_framework(LOG_DEBUG,
4194             "Graph engine: Reloading general properties for %s.\n", fmri);
4195 
4196         MUTEX_LOCK(&dgraph_lock);
4197 
4198         v = vertex_get_by_name(fmri);
4199         if (v == NULL || !(v->gv_flags & GV_CONFIGURED)) {
4200                 /* Will get the up-to-date properties. */
4201                 MUTEX_UNLOCK(&dgraph_lock);
4202                 err = dgraph_add_instance(fmri, inst, B_TRUE);
4203                 startd_free(fmri, max_scf_fmri_size);
4204                 scf_instance_destroy(inst);
4205                 return (err == ECANCELED ? 0 : err);
4206         }
4207 
4208         /* Read enabled & restarter from repository. */
4209         restarter_fmri = startd_alloc(max_scf_value_size);
4210         err = libscf_get_basic_instance_data(h, inst, v->gv_name, &enabled,
4211             &enabled_ovr, &restarter_fmri);
4212         if (err != 0 || enabled == -1) {
4213                 MUTEX_UNLOCK(&dgraph_lock);
4214                 scf_instance_destroy(inst);
4215                 startd_free(fmri, max_scf_fmri_size);
4216 
4217                 switch (err) {
4218                 case ENOENT:
4219                 case 0:
4220                         startd_free(restarter_fmri, max_scf_value_size);
4221                         return (-1);
4222 
4223                 case ECONNABORTED:
4224                 case ECANCELED:
4225                         startd_free(restarter_fmri, max_scf_value_size);
4226                         return (err);
4227 
4228                 default:
4229                         bad_error("libscf_get_basic_instance_data", err);
4230                 }
4231         }
4232 
4233         oldflags = v->gv_flags;
4234         v->gv_flags = (v->gv_flags & ~GV_ENBLD_NOOVR) |
4235             (enabled ? GV_ENBLD_NOOVR : 0);
4236 
4237         if (enabled_ovr != -1)
4238                 enabled = enabled_ovr;
4239 
4240         /*
4241          * If GV_ENBLD_NOOVR has changed, then we need to re-evaluate the
4242          * subgraph.
4243          */
4244         if (milestone > MILESTONE_NONE && v->gv_flags != oldflags)
4245                 (void) eval_subgraph(v, h);
4246 
4247         scf_instance_destroy(inst);
4248 
4249         /* Ignore restarter change for now. */
4250 
4251         startd_free(restarter_fmri, max_scf_value_size);
4252         startd_free(fmri, max_scf_fmri_size);
4253 
4254         /*
4255          * Always send _ENABLE or _DISABLE.  We could avoid this if the
4256          * restarter didn't change and the enabled value didn't change, but
4257          * that's not easy to check and improbable anyway, so we'll just do
4258          * this.
4259          */
4260         graph_enable_by_vertex(v, enabled, 1);
4261 
4262         MUTEX_UNLOCK(&dgraph_lock);
4263 
4264         return (0);
4265 }
4266 
4267 /*
4268  * Delete all of the property group dependencies of v, update inst's running
4269  * snapshot, and add the dependencies in the new snapshot.  If any of the new
4270  * dependencies would create a cycle, send _ADMIN_MAINT_ON.  Otherwise
4271  * reevaluate v's dependencies, send _START or _STOP as appropriate, and do
4272  * the same for v's dependents.
4273  *
4274  * Returns
4275  *   0 - success
4276  *   ECONNABORTED - repository connection broken
4277  *   ECANCELED - inst was deleted
4278  *   EINVAL - inst is invalid (e.g., missing general/enabled)
4279  *   -1 - libscf_snapshots_refresh() failed
4280  */
4281 static int
4282 dgraph_refresh_instance(graph_vertex_t *v, scf_instance_t *inst)
4283 {
4284         int r;
4285         int enabled;
4286         int32_t tset;
4287 
4288         assert(MUTEX_HELD(&dgraph_lock));
4289         assert(v->gv_type == GVT_INST);
4290 
4291         /* Only refresh services with valid general/enabled properties. */
4292         r = libscf_get_basic_instance_data(scf_instance_handle(inst), inst,
4293             v->gv_name, &enabled, NULL, NULL);
4294         switch (r) {
4295         case 0:
4296                 break;
4297 
4298         case ECONNABORTED:
4299         case ECANCELED:
4300                 return (r);
4301 
4302         case ENOENT:
4303                 log_framework(LOG_DEBUG,
4304                     "Ignoring %s because it has no general property group.\n",
4305                     v->gv_name);
4306                 return (EINVAL);
4307 
4308         default:
4309                 bad_error("libscf_get_basic_instance_data", r);
4310         }
4311 
4312         if ((tset = libscf_get_stn_tset(inst)) == -1) {
4313                 log_framework(LOG_WARNING,
4314                     "Failed to get notification parameters for %s: %s\n",
4315                     v->gv_name, scf_strerror(scf_error()));
4316                 tset = 0;
4317         }
4318         v->gv_stn_tset = tset;
4319         if (strcmp(v->gv_name, SCF_INSTANCE_GLOBAL) == 0)
4320                 stn_global = tset;
4321 
4322         if (enabled == -1)
4323                 return (EINVAL);
4324 
4325         r = libscf_snapshots_refresh(inst, v->gv_name);
4326         if (r != 0) {
4327                 if (r != -1)
4328                         bad_error("libscf_snapshots_refresh", r);
4329 
4330                 /* error logged */
4331                 return (r);
4332         }
4333 
4334         r = refresh_vertex(v, inst);
4335         if (r != 0 && r != ECONNABORTED)
4336                 bad_error("refresh_vertex", r);
4337         return (r);
4338 }
4339 
4340 /*
4341  * Returns true only if none of this service's dependents are 'up' -- online
4342  * or degraded (offline is considered down in this situation). This function
4343  * is somehow similar to is_nonsubgraph_leaf() but works on subtrees.
4344  */
4345 static boolean_t
4346 insubtree_dependents_down(graph_vertex_t *v)
4347 {
4348         graph_vertex_t *vv;
4349         graph_edge_t *e;
4350 
4351         assert(MUTEX_HELD(&dgraph_lock));
4352 
4353         for (e = uu_list_first(v->gv_dependents); e != NULL;
4354             e = uu_list_next(v->gv_dependents, e)) {
4355                 vv = e->ge_vertex;
4356                 if (vv->gv_type == GVT_INST) {
4357                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4358                                 continue;
4359 
4360                         if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4361                                 continue;
4362 
4363                         if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4364                             (vv->gv_state == RESTARTER_STATE_DEGRADED))
4365                                 return (B_FALSE);
4366                 } else {
4367                         /*
4368                          * Skip all excluded dependents and decide whether
4369                          * to offline the service based on the restart_on
4370                          * attribute.
4371                          */
4372                         if (is_depgrp_bypassed(vv))
4373                                 continue;
4374 
4375                         /*
4376                          * For dependency groups or service vertices, keep
4377                          * traversing to see if instances are running.
4378                          */
4379                         if (insubtree_dependents_down(vv) == B_FALSE)
4380                                 return (B_FALSE);
4381                 }
4382         }
4383 
4384         return (B_TRUE);
4385 }
4386 
4387 /*
4388  * Returns true only if none of this service's dependents are 'up' -- online,
4389  * degraded, or offline.
4390  */
4391 static int
4392 is_nonsubgraph_leaf(graph_vertex_t *v)
4393 {
4394         graph_vertex_t *vv;
4395         graph_edge_t *e;
4396 
4397         assert(MUTEX_HELD(&dgraph_lock));
4398 
4399         for (e = uu_list_first(v->gv_dependents);
4400             e != NULL;
4401             e = uu_list_next(v->gv_dependents, e)) {
4402 
4403                 vv = e->ge_vertex;
4404                 if (vv->gv_type == GVT_INST) {
4405                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4406                                 continue;
4407 
4408                         if (vv->gv_flags & GV_INSUBGRAPH)
4409                                 continue;
4410 
4411                         if (up_state(vv->gv_state))
4412                                 return (0);
4413                 } else {
4414                         /*
4415                          * For dependency group or service vertices, keep
4416                          * traversing to see if instances are running.
4417                          *
4418                          * We should skip exclude_all dependencies otherwise
4419                          * the vertex will never be considered as a leaf
4420                          * if the dependent is offline. The main reason for
4421                          * this is that disable_nonsubgraph_leaves() skips
4422                          * exclusion dependencies.
4423                          */
4424                         if (vv->gv_type == GVT_GROUP &&
4425                             vv->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4426                                 continue;
4427 
4428                         if (!is_nonsubgraph_leaf(vv))
4429                                 return (0);
4430                 }
4431         }
4432 
4433         return (1);
4434 }
4435 
4436 /*
4437  * Disable v temporarily.  Attempt to do this by setting its enabled override
4438  * property in the repository.  If that fails, send a _DISABLE command.
4439  * Returns 0 on success and ECONNABORTED if the repository connection is
4440  * broken.
4441  */
4442 static int
4443 disable_service_temporarily(graph_vertex_t *v, scf_handle_t *h)
4444 {
4445         const char * const emsg = "Could not temporarily disable %s because "
4446             "%s.  Will stop service anyways.  Repository status for the "
4447             "service may be inaccurate.\n";
4448         const char * const emsg_cbroken =
4449             "the repository connection was broken";
4450 
4451         scf_instance_t *inst;
4452         int r;
4453 
4454         inst = scf_instance_create(h);
4455         if (inst == NULL) {
4456                 char buf[100];
4457 
4458                 (void) snprintf(buf, sizeof (buf),
4459                     "scf_instance_create() failed (%s)",
4460                     scf_strerror(scf_error()));
4461                 log_error(LOG_WARNING, emsg, v->gv_name, buf);
4462 
4463                 graph_enable_by_vertex(v, 0, 0);
4464                 return (0);
4465         }
4466 
4467         r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
4468             NULL, NULL, SCF_DECODE_FMRI_EXACT);
4469         if (r != 0) {
4470                 switch (scf_error()) {
4471                 case SCF_ERROR_CONNECTION_BROKEN:
4472                         log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4473                         graph_enable_by_vertex(v, 0, 0);
4474                         return (ECONNABORTED);
4475 
4476                 case SCF_ERROR_NOT_FOUND:
4477                         return (0);
4478 
4479                 case SCF_ERROR_HANDLE_MISMATCH:
4480                 case SCF_ERROR_INVALID_ARGUMENT:
4481                 case SCF_ERROR_CONSTRAINT_VIOLATED:
4482                 case SCF_ERROR_NOT_BOUND:
4483                 default:
4484                         bad_error("scf_handle_decode_fmri",
4485                             scf_error());
4486                 }
4487         }
4488 
4489         r = libscf_set_enable_ovr(inst, 0);
4490         switch (r) {
4491         case 0:
4492                 scf_instance_destroy(inst);
4493                 return (0);
4494 
4495         case ECANCELED:
4496                 scf_instance_destroy(inst);
4497                 return (0);
4498 
4499         case ECONNABORTED:
4500                 log_error(LOG_WARNING, emsg, v->gv_name, emsg_cbroken);
4501                 graph_enable_by_vertex(v, 0, 0);
4502                 return (ECONNABORTED);
4503 
4504         case EPERM:
4505                 log_error(LOG_WARNING, emsg, v->gv_name,
4506                     "the repository denied permission");
4507                 graph_enable_by_vertex(v, 0, 0);
4508                 return (0);
4509 
4510         case EROFS:
4511                 log_error(LOG_WARNING, emsg, v->gv_name,
4512                     "the repository is read-only");
4513                 graph_enable_by_vertex(v, 0, 0);
4514                 return (0);
4515 
4516         default:
4517                 bad_error("libscf_set_enable_ovr", r);
4518                 /* NOTREACHED */
4519         }
4520 }
4521 
4522 /*
4523  * Of the transitive instance dependencies of v, offline those which are
4524  * in the subtree and which are leaves (i.e., have no dependents which are
4525  * "up").
4526  */
4527 void
4528 offline_subtree_leaves(graph_vertex_t *v, void *arg)
4529 {
4530         assert(MUTEX_HELD(&dgraph_lock));
4531 
4532         /* If v isn't an instance, recurse on its dependencies. */
4533         if (v->gv_type != GVT_INST) {
4534                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4535                 return;
4536         }
4537 
4538         /*
4539          * If v is not in the subtree, so should all of its dependencies,
4540          * so do nothing.
4541          */
4542         if ((v->gv_flags & GV_TOOFFLINE) == 0)
4543                 return;
4544 
4545         /* If v isn't a leaf because it's already down, recurse. */
4546         if (!up_state(v->gv_state)) {
4547                 graph_walk_dependencies(v, offline_subtree_leaves, arg);
4548                 return;
4549         }
4550 
4551         /* if v is a leaf, offline it or disable it if it's the last one */
4552         if (insubtree_dependents_down(v) == B_TRUE) {
4553                 if (v->gv_flags & GV_TODISABLE)
4554                         vertex_send_event(v,
4555                             RESTARTER_EVENT_TYPE_ADMIN_DISABLE);
4556                 else
4557                         offline_vertex(v);
4558         }
4559 }
4560 
4561 void
4562 graph_offline_subtree_leaves(graph_vertex_t *v, void *h)
4563 {
4564         graph_walk_dependencies(v, offline_subtree_leaves, (void *)h);
4565 }
4566 
4567 
4568 /*
4569  * Of the transitive instance dependencies of v, disable those which are not
4570  * in the subgraph and which are leaves (i.e., have no dependents which are
4571  * "up").
4572  */
4573 static void
4574 disable_nonsubgraph_leaves(graph_vertex_t *v, void *arg)
4575 {
4576         assert(MUTEX_HELD(&dgraph_lock));
4577 
4578         /*
4579          * We must skip exclusion dependencies because they are allowed to
4580          * complete dependency cycles.  This is correct because A's exclusion
4581          * dependency on B doesn't bear on the order in which they should be
4582          * stopped.  Indeed, the exclusion dependency should guarantee that
4583          * they are never online at the same time.
4584          */
4585         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
4586                 return;
4587 
4588         /* If v isn't an instance, recurse on its dependencies. */
4589         if (v->gv_type != GVT_INST)
4590                 goto recurse;
4591 
4592         if ((v->gv_flags & GV_CONFIGURED) == 0)
4593                 /*
4594                  * Unconfigured instances should have no dependencies, but in
4595                  * case they ever get them,
4596                  */
4597                 goto recurse;
4598 
4599         /*
4600          * If v is in the subgraph, so should all of its dependencies, so do
4601          * nothing.
4602          */
4603         if (v->gv_flags & GV_INSUBGRAPH)
4604                 return;
4605 
4606         /* If v isn't a leaf because it's already down, recurse. */
4607         if (!up_state(v->gv_state))
4608                 goto recurse;
4609 
4610         /* If v is disabled but not down yet, be patient. */
4611         if ((v->gv_flags & GV_ENABLED) == 0)
4612                 return;
4613 
4614         /* If v is a leaf, disable it. */
4615         if (is_nonsubgraph_leaf(v))
4616                 (void) disable_service_temporarily(v, (scf_handle_t *)arg);
4617 
4618         return;
4619 
4620 recurse:
4621         graph_walk_dependencies(v, disable_nonsubgraph_leaves, arg);
4622 }
4623 
4624 static int
4625 stn_restarter_state(restarter_instance_state_t rstate)
4626 {
4627         static const struct statemap {
4628                 restarter_instance_state_t restarter_state;
4629                 int scf_state;
4630         } map[] = {
4631                 { RESTARTER_STATE_UNINIT, SCF_STATE_UNINIT },
4632                 { RESTARTER_STATE_MAINT, SCF_STATE_MAINT },
4633                 { RESTARTER_STATE_OFFLINE, SCF_STATE_OFFLINE },
4634                 { RESTARTER_STATE_DISABLED, SCF_STATE_DISABLED },
4635                 { RESTARTER_STATE_ONLINE, SCF_STATE_ONLINE },
4636                 { RESTARTER_STATE_DEGRADED, SCF_STATE_DEGRADED }
4637         };
4638 
4639         int i;
4640 
4641         for (i = 0; i < sizeof (map) / sizeof (map[0]); i++) {
4642                 if (rstate == map[i].restarter_state)
4643                         return (map[i].scf_state);
4644         }
4645 
4646         return (-1);
4647 }
4648 
4649 /*
4650  * State transition counters
4651  * Not incremented atomically - indicative only
4652  */
4653 static uint64_t stev_ct_maint;
4654 static uint64_t stev_ct_hwerr;
4655 static uint64_t stev_ct_service;
4656 static uint64_t stev_ct_global;
4657 static uint64_t stev_ct_noprefs;
4658 static uint64_t stev_ct_from_uninit;
4659 static uint64_t stev_ct_bad_state;
4660 static uint64_t stev_ct_ovr_prefs;
4661 
4662 static void
4663 dgraph_state_transition_notify(graph_vertex_t *v,
4664     restarter_instance_state_t old_state, restarter_str_t reason)
4665 {
4666         restarter_instance_state_t new_state = v->gv_state;
4667         int stn_transition, maint;
4668         int from, to;
4669         nvlist_t *attr;
4670         fmev_pri_t pri = FMEV_LOPRI;
4671         int raise = 0;
4672 
4673         if ((from = stn_restarter_state(old_state)) == -1 ||
4674             (to = stn_restarter_state(new_state)) == -1) {
4675                 stev_ct_bad_state++;
4676                 return;
4677         }
4678 
4679         stn_transition = from << 16 | to;
4680 
4681         maint = (to == SCF_STATE_MAINT || from == SCF_STATE_MAINT);
4682 
4683         if (maint) {
4684                 /*
4685                  * All transitions to/from maintenance state must raise
4686                  * an event.
4687                  */
4688                 raise++;
4689                 pri = FMEV_HIPRI;
4690                 stev_ct_maint++;
4691         } else if (reason == restarter_str_ct_ev_hwerr) {
4692                 /*
4693                  * All transitions caused by hardware fault must raise
4694                  * an event
4695                  */
4696                 raise++;
4697                 pri = FMEV_HIPRI;
4698                 stev_ct_hwerr++;
4699         } else if (stn_transition & v->gv_stn_tset) {
4700                 /*
4701                  * Specifically enabled event.
4702                  */
4703                 raise++;
4704                 stev_ct_service++;
4705         } else if (from == SCF_STATE_UNINIT) {
4706                 /*
4707                  * Only raise these if specifically selected above.
4708                  */
4709                 stev_ct_from_uninit++;
4710         } else if (stn_transition & stn_global &&
4711             (IS_ENABLED(v) == 1 || to == SCF_STATE_DISABLED)) {
4712                 raise++;
4713                 stev_ct_global++;
4714         } else {
4715                 stev_ct_noprefs++;
4716         }
4717 
4718         if (info_events_all) {
4719                 stev_ct_ovr_prefs++;
4720                 raise++;
4721         }
4722         if (!raise)
4723                 return;
4724 
4725         if (nvlist_alloc(&attr, NV_UNIQUE_NAME, 0) != 0 ||
4726             nvlist_add_string(attr, "fmri", v->gv_name) != 0 ||
4727             nvlist_add_uint32(attr, "reason-version",
4728             restarter_str_version()) || nvlist_add_string(attr, "reason-short",
4729             restarter_get_str_short(reason)) != 0 ||
4730             nvlist_add_string(attr, "reason-long",
4731             restarter_get_str_long(reason)) != 0 ||
4732             nvlist_add_int32(attr, "transition", stn_transition) != 0) {
4733                 log_framework(LOG_WARNING,
4734                     "FMEV: %s could not create nvlist for transition "
4735                     "event: %s\n", v->gv_name, strerror(errno));
4736                 nvlist_free(attr);
4737                 return;
4738         }
4739 
4740         if (fmev_rspublish_nvl(FMEV_RULESET_SMF, "state-transition",
4741             instance_state_str[new_state], pri, attr) != FMEV_SUCCESS) {
4742                 log_framework(LOG_DEBUG,
4743                     "FMEV: %s failed to publish transition event: %s\n",
4744                     v->gv_name, fmev_strerror(fmev_errno));
4745                 nvlist_free(attr);
4746         }
4747 }
4748 
4749 /*
4750  * Find the vertex for inst_name.  If it doesn't exist, return ENOENT.
4751  * Otherwise set its state to state.  If the instance has entered a state
4752  * which requires automatic action, take it (Uninitialized: do
4753  * dgraph_refresh_instance() without the snapshot update.  Disabled: if the
4754  * instance should be enabled, send _ENABLE.  Offline: if the instance should
4755  * be disabled, send _DISABLE, and if its dependencies are satisfied, send
4756  * _START.  Online, Degraded: if the instance wasn't running, update its start
4757  * snapshot.  Maintenance: no action.)
4758  *
4759  * Also fails with ECONNABORTED, or EINVAL if state is invalid.
4760  */
4761 static int
4762 dgraph_set_instance_state(scf_handle_t *h, const char *inst_name,
4763     protocol_states_t *states)
4764 {
4765         graph_vertex_t *v;
4766         int err = 0;
4767         restarter_instance_state_t old_state;
4768         restarter_instance_state_t state = states->ps_state;
4769         restarter_error_t serr = states->ps_err;
4770 
4771         MUTEX_LOCK(&dgraph_lock);
4772 
4773         v = vertex_get_by_name(inst_name);
4774         if (v == NULL) {
4775                 MUTEX_UNLOCK(&dgraph_lock);
4776                 return (ENOENT);
4777         }
4778 
4779         assert(v->gv_type == GVT_INST);
4780 
4781         switch (state) {
4782         case RESTARTER_STATE_UNINIT:
4783         case RESTARTER_STATE_DISABLED:
4784         case RESTARTER_STATE_OFFLINE:
4785         case RESTARTER_STATE_ONLINE:
4786         case RESTARTER_STATE_DEGRADED:
4787         case RESTARTER_STATE_MAINT:
4788                 break;
4789 
4790         default:
4791                 MUTEX_UNLOCK(&dgraph_lock);
4792                 return (EINVAL);
4793         }
4794 
4795         log_framework(LOG_DEBUG, "Graph noting %s %s -> %s.\n", v->gv_name,
4796             instance_state_str[v->gv_state], instance_state_str[state]);
4797 
4798         old_state = v->gv_state;
4799         v->gv_state = state;
4800 
4801         v->gv_reason = states->ps_reason;
4802         err = gt_transition(h, v, serr, old_state);
4803         if (err == 0 && v->gv_state != old_state) {
4804                 dgraph_state_transition_notify(v, old_state, states->ps_reason);
4805         }
4806 
4807         MUTEX_UNLOCK(&dgraph_lock);
4808         return (err);
4809 }
4810 
4811 /*
4812  * Handle state changes during milestone shutdown.  See
4813  * dgraph_set_milestone().  If the repository connection is broken,
4814  * ECONNABORTED will be returned, though a _DISABLE command will be sent for
4815  * the vertex anyway.
4816  */
4817 int
4818 vertex_subgraph_dependencies_shutdown(scf_handle_t *h, graph_vertex_t *v,
4819     restarter_instance_state_t old_state)
4820 {
4821         int was_up, now_up;
4822         int ret = 0;
4823 
4824         assert(v->gv_type == GVT_INST);
4825 
4826         /* Don't care if we're not going to a milestone. */
4827         if (milestone == NULL)
4828                 return (0);
4829 
4830         /* Don't care if we already finished coming down. */
4831         if (non_subgraph_svcs == 0)
4832                 return (0);
4833 
4834         /* Don't care if the service is in the subgraph. */
4835         if (v->gv_flags & GV_INSUBGRAPH)
4836                 return (0);
4837 
4838         /*
4839          * Update non_subgraph_svcs.  It is the number of non-subgraph
4840          * services which are in online, degraded, or offline.
4841          */
4842 
4843         was_up = up_state(old_state);
4844         now_up = up_state(v->gv_state);
4845 
4846         if (!was_up && now_up) {
4847                 ++non_subgraph_svcs;
4848         } else if (was_up && !now_up) {
4849                 --non_subgraph_svcs;
4850 
4851                 if (non_subgraph_svcs == 0) {
4852                         if (halting != -1) {
4853                                 do_uadmin();
4854                         } else if (go_single_user_mode || go_to_level1) {
4855                                 (void) startd_thread_create(single_user_thread,
4856                                     NULL);
4857                         }
4858                         return (0);
4859                 }
4860         }
4861 
4862         /* If this service is a leaf, it should be disabled. */
4863         if ((v->gv_flags & GV_ENABLED) && is_nonsubgraph_leaf(v)) {
4864                 int r;
4865 
4866                 r = disable_service_temporarily(v, h);
4867                 switch (r) {
4868                 case 0:
4869                         break;
4870 
4871                 case ECONNABORTED:
4872                         ret = ECONNABORTED;
4873                         break;
4874 
4875                 default:
4876                         bad_error("disable_service_temporarily", r);
4877                 }
4878         }
4879 
4880         /*
4881          * If the service just came down, propagate the disable to the newly
4882          * exposed leaves.
4883          */
4884         if (was_up && !now_up)
4885                 graph_walk_dependencies(v, disable_nonsubgraph_leaves,
4886                     (void *)h);
4887 
4888         return (ret);
4889 }
4890 
4891 /*
4892  * Decide whether to start up an sulogin thread after a service is
4893  * finished changing state.  Only need to do the full can_come_up()
4894  * evaluation if an instance is changing state, we're not halfway through
4895  * loading the thread, and we aren't shutting down or going to the single
4896  * user milestone.
4897  */
4898 void
4899 graph_transition_sulogin(restarter_instance_state_t state,
4900     restarter_instance_state_t old_state)
4901 {
4902         assert(MUTEX_HELD(&dgraph_lock));
4903 
4904         if (state != old_state && st->st_load_complete &&
4905             !go_single_user_mode && !go_to_level1 &&
4906             halting == -1) {
4907                 if (!sulogin_thread_running && !can_come_up()) {
4908                         (void) startd_thread_create(sulogin_thread, NULL);
4909                         sulogin_thread_running = B_TRUE;
4910                 }
4911         }
4912 }
4913 
4914 /*
4915  * Propagate a start, stop event, or a satisfiability event.
4916  *
4917  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4918  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4919  * full dependent graph to check for newly satisfied nodes.  This is
4920  * necessary for cases when non-direct dependents may be effected but direct
4921  * dependents may not (e.g. for optional_all evaluations, see the
4922  * propagate_satbility() comments).
4923  *
4924  * PROPAGATE_SAT should be used whenever a non-running service moves into
4925  * a state which can satisfy optional dependencies, like disabled or
4926  * maintenance.
4927  */
4928 void
4929 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4930     restarter_error_t rerr)
4931 {
4932         if (type == PROPAGATE_STOP) {
4933                 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4934         } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4935                 graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
4936 
4937                 if (type == PROPAGATE_SAT)
4938                         propagate_satbility(v);
4939         } else {
4940 #ifndef NDEBUG
4941                 uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4942                     __LINE__, type);
4943 #endif
4944                 abort();
4945         }
4946 }
4947 
4948 /*
4949  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4950  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4951  * all property group dependencies, and the dependency on the restarter,
4952  * disposing of vertices as appropriate.  If other vertices depend on this
4953  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4954  * returns 0.
4955  */
4956 static int
4957 dgraph_remove_instance(const char *fmri, scf_handle_t *h)
4958 {
4959         graph_vertex_t *v;
4960         graph_edge_t *e;
4961         uu_list_t *old_deps;
4962         int err;
4963 
4964         log_framework(LOG_DEBUG, "Graph engine: Removing %s.\n", fmri);
4965 
4966         MUTEX_LOCK(&dgraph_lock);
4967 
4968         v = vertex_get_by_name(fmri);
4969         if (v == NULL) {
4970                 MUTEX_UNLOCK(&dgraph_lock);
4971                 return (0);
4972         }
4973 
4974         /* Send restarter delete event. */
4975         if (v->gv_flags & GV_CONFIGURED)
4976                 graph_unset_restarter(v);
4977 
4978         if (milestone > MILESTONE_NONE) {
4979                 /*
4980                  * Make a list of v's current dependencies so we can
4981                  * reevaluate their GV_INSUBGRAPH flags after the dependencies
4982                  * are removed.
4983                  */
4984                 old_deps = startd_list_create(graph_edge_pool, NULL, 0);
4985 
4986                 err = uu_list_walk(v->gv_dependencies,
4987                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
4988                 assert(err == 0);
4989         }
4990 
4991         delete_instance_dependencies(v, B_TRUE);
4992 
4993         /*
4994          * Deleting an instance can both satisfy and unsatisfy dependencies,
4995          * depending on their type.  First propagate the stop as a RERR_RESTART
4996          * event -- deletion isn't a fault, just a normal stop.  This gives
4997          * dependent services the chance to do a clean shutdown.  Then, mark
4998          * the service as unconfigured and propagate the start event for the
4999          * optional_all dependencies that might have become satisfied.
5000          */
5001         graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
5002 
5003         v->gv_flags &= ~GV_CONFIGURED;
5004         v->gv_flags &= ~GV_DEATHROW;
5005 
5006         graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
5007         propagate_satbility(v);
5008 
5009         /*
5010          * If there are no (non-service) dependents, the vertex can be
5011          * completely removed.
5012          */
5013         if (v != milestone && v->gv_refs == 0 &&
5014             uu_list_numnodes(v->gv_dependents) == 1)
5015                 remove_inst_vertex(v);
5016 
5017         if (milestone > MILESTONE_NONE) {
5018                 void *cookie = NULL;
5019 
5020                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5021                         v = e->ge_vertex;
5022 
5023                         if (vertex_unref(v) == VERTEX_INUSE)
5024                                 while (eval_subgraph(v, h) == ECONNABORTED)
5025                                         libscf_handle_rebind(h);
5026 
5027                         startd_free(e, sizeof (*e));
5028                 }
5029 
5030                 uu_list_destroy(old_deps);
5031         }
5032 
5033         MUTEX_UNLOCK(&dgraph_lock);
5034 
5035         return (0);
5036 }
5037 
5038 /*
5039  * Return the eventual (maybe current) milestone in the form of a
5040  * legacy runlevel.
5041  */
5042 static char
5043 target_milestone_as_runlevel()
5044 {
5045         assert(MUTEX_HELD(&dgraph_lock));
5046 
5047         if (milestone == NULL)
5048                 return ('3');
5049         else if (milestone == MILESTONE_NONE)
5050                 return ('0');
5051 
5052         if (strcmp(milestone->gv_name, multi_user_fmri) == 0)
5053                 return ('2');
5054         else if (strcmp(milestone->gv_name, single_user_fmri) == 0)
5055                 return ('S');
5056         else if (strcmp(milestone->gv_name, multi_user_svr_fmri) == 0)
5057                 return ('3');
5058 
5059 #ifndef NDEBUG
5060         (void) fprintf(stderr, "%s:%d: Unknown milestone name \"%s\".\n",
5061             __FILE__, __LINE__, milestone->gv_name);
5062 #endif
5063         abort();
5064         /* NOTREACHED */
5065 }
5066 
5067 static struct {
5068         char    rl;
5069         int     sig;
5070 } init_sigs[] = {
5071         { 'S', SIGBUS },
5072         { '0', SIGINT },
5073         { '1', SIGQUIT },
5074         { '2', SIGILL },
5075         { '3', SIGTRAP },
5076         { '4', SIGIOT },
5077         { '5', SIGEMT },
5078         { '6', SIGFPE },
5079         { 0, 0 }
5080 };
5081 
5082 static void
5083 signal_init(char rl)
5084 {
5085         pid_t init_pid;
5086         int i;
5087 
5088         assert(MUTEX_HELD(&dgraph_lock));
5089 
5090         if (zone_getattr(getzoneid(), ZONE_ATTR_INITPID, &init_pid,
5091             sizeof (init_pid)) != sizeof (init_pid)) {
5092                 log_error(LOG_NOTICE, "Could not get pid to signal init.\n");
5093                 return;
5094         }
5095 
5096         for (i = 0; init_sigs[i].rl != 0; ++i)
5097                 if (init_sigs[i].rl == rl)
5098                         break;
5099 
5100         if (init_sigs[i].rl != 0) {
5101                 if (kill(init_pid, init_sigs[i].sig) != 0) {
5102                         switch (errno) {
5103                         case EPERM:
5104                         case ESRCH:
5105                                 log_error(LOG_NOTICE, "Could not signal init: "
5106                                     "%s.\n", strerror(errno));
5107                                 break;
5108 
5109                         case EINVAL:
5110                         default:
5111                                 bad_error("kill", errno);
5112                         }
5113                 }
5114         }
5115 }
5116 
5117 /*
5118  * This is called when one of the major milestones changes state, or when
5119  * init is signalled and tells us it was told to change runlevel.  We wait
5120  * to reach the milestone because this allows /etc/inittab entries to retain
5121  * some boot ordering: historically, entries could place themselves before/after
5122  * the running of /sbin/rcX scripts but we can no longer make the
5123  * distinction because the /sbin/rcX scripts no longer exist as punctuation
5124  * marks in /etc/inittab.
5125  *
5126  * Also, we only trigger an update when we reach the eventual target
5127  * milestone: without this, an /etc/inittab entry marked only for
5128  * runlevel 2 would be executed for runlevel 3, which is not how
5129  * /etc/inittab entries work.
5130  *
5131  * If we're single user coming online, then we set utmpx to the target
5132  * runlevel so that legacy scripts can work as expected.
5133  */
5134 static void
5135 graph_runlevel_changed(char rl, int online)
5136 {
5137         char trl;
5138 
5139         assert(MUTEX_HELD(&dgraph_lock));
5140 
5141         trl = target_milestone_as_runlevel();
5142 
5143         if (online) {
5144                 if (rl == trl) {
5145                         current_runlevel = trl;
5146                         signal_init(trl);
5147                 } else if (rl == 'S') {
5148                         /*
5149                          * At boot, set the entry early for the benefit of the
5150                          * legacy init scripts.
5151                          */
5152                         utmpx_set_runlevel(trl, 'S', B_FALSE);
5153                 }
5154         } else {
5155                 if (rl == '3' && trl == '2') {
5156                         current_runlevel = trl;
5157                         signal_init(trl);
5158                 } else if (rl == '2' && trl == 'S') {
5159                         current_runlevel = trl;
5160                         signal_init(trl);
5161                 }
5162         }
5163 }
5164 
5165 /*
5166  * Move to a backwards-compatible runlevel by executing the appropriate
5167  * /etc/rc?.d/K* scripts and/or setting the milestone.
5168  *
5169  * Returns
5170  *   0 - success
5171  *   ECONNRESET - success, but handle was reset
5172  *   ECONNABORTED - repository connection broken
5173  *   ECANCELED - pg was deleted
5174  */
5175 static int
5176 dgraph_set_runlevel(scf_propertygroup_t *pg, scf_property_t *prop)
5177 {
5178         char rl;
5179         scf_handle_t *h;
5180         int r;
5181         const char *ms = NULL;  /* what to commit as options/milestone */
5182         boolean_t rebound = B_FALSE;
5183         int mark_rl = 0;
5184 
5185         const char * const stop = "stop";
5186 
5187         r = libscf_extract_runlevel(prop, &rl);
5188         switch (r) {
5189         case 0:
5190                 break;
5191 
5192         case ECONNABORTED:
5193         case ECANCELED:
5194                 return (r);
5195 
5196         case EINVAL:
5197         case ENOENT:
5198                 log_error(LOG_WARNING, "runlevel property is misconfigured; "
5199                     "ignoring.\n");
5200                 /* delete the bad property */
5201                 goto nolock_out;
5202 
5203         default:
5204                 bad_error("libscf_extract_runlevel", r);
5205         }
5206 
5207         switch (rl) {
5208         case 's':
5209                 rl = 'S';
5210                 /* FALLTHROUGH */
5211 
5212         case 'S':
5213         case '2':
5214         case '3':
5215                 /*
5216                  * These cases cause a milestone change, so
5217                  * graph_runlevel_changed() will eventually deal with
5218                  * signalling init.
5219                  */
5220                 break;
5221 
5222         case '0':
5223         case '1':
5224         case '4':
5225         case '5':
5226         case '6':
5227                 mark_rl = 1;
5228                 break;
5229 
5230         default:
5231                 log_framework(LOG_NOTICE, "Unknown runlevel '%c'.\n", rl);
5232                 ms = NULL;
5233                 goto nolock_out;
5234         }
5235 
5236         h = scf_pg_handle(pg);
5237 
5238         MUTEX_LOCK(&dgraph_lock);
5239 
5240         /*
5241          * Since this triggers no milestone changes, force it by hand.
5242          */
5243         if (current_runlevel == '4' && rl == '3')
5244                 mark_rl = 1;
5245 
5246         /*
5247          * 1. If we are here after an "init X":
5248          *
5249          * init X
5250          *      init/lscf_set_runlevel()
5251          *              process_pg_event()
5252          *              dgraph_set_runlevel()
5253          *
5254          * then we haven't passed through graph_runlevel_changed() yet,
5255          * therefore 'current_runlevel' has not changed for sure but 'rl' has.
5256          * In consequence, if 'rl' is lower than 'current_runlevel', we change
5257          * the system runlevel and execute the appropriate /etc/rc?.d/K* scripts
5258          * past this test.
5259          *
5260          * 2. On the other hand, if we are here after a "svcadm milestone":
5261          *
5262          * svcadm milestone X
5263          *      dgraph_set_milestone()
5264          *              handle_graph_update_event()
5265          *              dgraph_set_instance_state()
5266          *              graph_post_X_[online|offline]()
5267          *              graph_runlevel_changed()
5268          *              signal_init()
5269          *                      init/lscf_set_runlevel()
5270          *                              process_pg_event()
5271          *                              dgraph_set_runlevel()
5272          *
5273          * then we already passed through graph_runlevel_changed() (by the way
5274          * of dgraph_set_milestone()) and 'current_runlevel' may have changed
5275          * and already be equal to 'rl' so we are going to return immediately
5276          * from dgraph_set_runlevel() without changing the system runlevel and
5277          * without executing the /etc/rc?.d/K* scripts.
5278          */
5279         if (rl == current_runlevel) {
5280                 ms = NULL;
5281                 goto out;
5282         }
5283 
5284         log_framework(LOG_DEBUG, "Changing to runlevel '%c'.\n", rl);
5285 
5286         /*
5287          * Make sure stop rc scripts see the new settings via who -r.
5288          */
5289         utmpx_set_runlevel(rl, current_runlevel, B_TRUE);
5290 
5291         /*
5292          * Some run levels don't have a direct correspondence to any
5293          * milestones, so we have to signal init directly.
5294          */
5295         if (mark_rl) {
5296                 current_runlevel = rl;
5297                 signal_init(rl);
5298         }
5299 
5300         switch (rl) {
5301         case 'S':
5302                 uu_warn("The system is coming down for administration.  "
5303                     "Please wait.\n");
5304                 fork_rc_script(rl, stop, B_FALSE);
5305                 ms = single_user_fmri;
5306                 go_single_user_mode = B_TRUE;
5307                 break;
5308 
5309         case '0':
5310                 halting_time = time(NULL);
5311                 fork_rc_script(rl, stop, B_TRUE);
5312                 halting = AD_HALT;
5313                 goto uadmin;
5314 
5315         case '5':
5316                 halting_time = time(NULL);
5317                 fork_rc_script(rl, stop, B_TRUE);
5318                 halting = AD_POWEROFF;
5319                 goto uadmin;
5320 
5321         case '6':
5322                 halting_time = time(NULL);
5323                 fork_rc_script(rl, stop, B_TRUE);
5324                 if (scf_is_fastboot_default() && getzoneid() == GLOBAL_ZONEID)
5325                         halting = AD_FASTREBOOT;
5326                 else
5327                         halting = AD_BOOT;
5328 
5329 uadmin:
5330                 uu_warn("The system is coming down.  Please wait.\n");
5331                 ms = "none";
5332 
5333                 /*
5334                  * We can't wait until all services are offline since this
5335                  * thread is responsible for taking them offline.  Instead we
5336                  * set halting to the second argument for uadmin() and call
5337                  * do_uadmin() from dgraph_set_instance_state() when
5338                  * appropriate.
5339                  */
5340                 break;
5341 
5342         case '1':
5343                 if (current_runlevel != 'S') {
5344                         uu_warn("Changing to state 1.\n");
5345                         fork_rc_script(rl, stop, B_FALSE);
5346                 } else {
5347                         uu_warn("The system is coming up for administration.  "
5348                             "Please wait.\n");
5349                 }
5350                 ms = single_user_fmri;
5351                 go_to_level1 = B_TRUE;
5352                 break;
5353 
5354         case '2':
5355                 if (current_runlevel == '3' || current_runlevel == '4')
5356                         fork_rc_script(rl, stop, B_FALSE);
5357                 ms = multi_user_fmri;
5358                 break;
5359 
5360         case '3':
5361         case '4':
5362                 ms = "all";
5363                 break;
5364 
5365         default:
5366 #ifndef NDEBUG
5367                 (void) fprintf(stderr, "%s:%d: Uncaught case %d ('%c').\n",
5368                     __FILE__, __LINE__, rl, rl);
5369 #endif
5370                 abort();
5371         }
5372 
5373 out:
5374         MUTEX_UNLOCK(&dgraph_lock);
5375 
5376 nolock_out:
5377         switch (r = libscf_clear_runlevel(pg, ms)) {
5378         case 0:
5379                 break;
5380 
5381         case ECONNABORTED:
5382                 libscf_handle_rebind(h);
5383                 rebound = B_TRUE;
5384                 goto nolock_out;
5385 
5386         case ECANCELED:
5387                 break;
5388 
5389         case EPERM:
5390         case EACCES:
5391         case EROFS:
5392                 log_error(LOG_NOTICE, "Could not delete \"%s/%s\" property: "
5393                     "%s.\n", SCF_PG_OPTIONS, "runlevel", strerror(r));
5394                 break;
5395 
5396         default:
5397                 bad_error("libscf_clear_runlevel", r);
5398         }
5399 
5400         return (rebound ? ECONNRESET : 0);
5401 }
5402 
5403 /*
5404  * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5405  * to the instances that are supposed to go offline during an
5406  * administrative disable operation.
5407  */
5408 static int
5409 mark_subtree(graph_edge_t *e, void *arg)
5410 {
5411         graph_vertex_t *v;
5412         int r;
5413 
5414         v = e->ge_vertex;
5415 
5416         /* If it's already in the subgraph, skip. */
5417         if (v->gv_flags & GV_TOOFFLINE)
5418                 return (UU_WALK_NEXT);
5419 
5420         switch (v->gv_type) {
5421         case GVT_INST:
5422                 /* If the instance is already offline, skip it. */
5423                 if (!inst_running(v))
5424                         return (UU_WALK_NEXT);
5425 
5426                 v->gv_flags |= GV_TOOFFLINE;
5427                 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5428                 break;
5429         case GVT_GROUP:
5430                 /*
5431                  * Skip all excluded dependents and decide whether to offline
5432                  * the service based on the restart_on attribute.
5433                  */
5434                 if (is_depgrp_bypassed(v))
5435                         return (UU_WALK_NEXT);
5436                 break;
5437         }
5438 
5439         r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5440             0);
5441         assert(r == 0);
5442         return (UU_WALK_NEXT);
5443 }
5444 
5445 static int
5446 mark_subgraph(graph_edge_t *e, void *arg)
5447 {
5448         graph_vertex_t *v;
5449         int r;
5450         int optional = (int)arg;
5451 
5452         v = e->ge_vertex;
5453 
5454         /* If it's already in the subgraph, skip. */
5455         if (v->gv_flags & GV_INSUBGRAPH)
5456                 return (UU_WALK_NEXT);
5457 
5458         /*
5459          * Keep track if walk has entered an optional dependency group
5460          */
5461         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_OPTIONAL_ALL) {
5462                 optional = 1;
5463         }
5464         /*
5465          * Quit if we are in an optional dependency group and the instance
5466          * is disabled
5467          */
5468         if (optional && (v->gv_type == GVT_INST) &&
5469             (!(v->gv_flags & GV_ENBLD_NOOVR)))
5470                 return (UU_WALK_NEXT);
5471 
5472         v->gv_flags |= GV_INSUBGRAPH;
5473 
5474         /* Skip all excluded dependencies. */
5475         if (v->gv_type == GVT_GROUP && v->gv_depgroup == DEPGRP_EXCLUDE_ALL)
5476                 return (UU_WALK_NEXT);
5477 
5478         r = uu_list_walk(v->gv_dependencies, (uu_walk_fn_t *)mark_subgraph,
5479             (void *)optional, 0);
5480         assert(r == 0);
5481         return (UU_WALK_NEXT);
5482 }
5483 
5484 /*
5485  * Bring down all services which are not dependencies of fmri.  The
5486  * dependencies of fmri (direct & indirect) will constitute the "subgraph",
5487  * and will have the GV_INSUBGRAPH flag set.  The rest must be brought down,
5488  * which means the state is "disabled", "maintenance", or "uninitialized".  We
5489  * could consider "offline" to be down, and refrain from sending start
5490  * commands for such services, but that's not strictly necessary, so we'll
5491  * decline to intrude on the state machine.  It would probably confuse users
5492  * anyway.
5493  *
5494  * The services should be brought down in reverse-dependency order, so we
5495  * can't do it all at once here.  We initiate by override-disabling the leaves
5496  * of the dependency tree -- those services which are up but have no
5497  * dependents which are up.  When they come down,
5498  * vertex_subgraph_dependencies_shutdown() will override-disable the newly
5499  * exposed leaves.  Perseverance will ensure completion.
5500  *
5501  * Sometimes we need to take action when the transition is complete, like
5502  * start sulogin or halt the system.  To tell when we're done, we initialize
5503  * non_subgraph_svcs here to be the number of services which need to come
5504  * down.  As each does, we decrement the counter.  When it hits zero, we take
5505  * the appropriate action.  See vertex_subgraph_dependencies_shutdown().
5506  *
5507  * In case we're coming up, we also remove any enable-overrides for the
5508  * services which are dependencies of fmri.
5509  *
5510  * If norepository is true, the function will not change the repository.
5511  *
5512  * The decision to change the system run level in accordance with the milestone
5513  * is taken in dgraph_set_runlevel().
5514  *
5515  * Returns
5516  *   0 - success
5517  *   ECONNRESET - success, but handle was rebound
5518  *   EINVAL - fmri is invalid (error is logged)
5519  *   EALREADY - the milestone is already set to fmri
5520  *   ENOENT - a configured vertex does not exist for fmri (an error is logged)
5521  */
5522 static int
5523 dgraph_set_milestone(const char *fmri, scf_handle_t *h, boolean_t norepository)
5524 {
5525         const char *cfmri, *fs;
5526         graph_vertex_t *nm, *v;
5527         int ret = 0, r;
5528         scf_instance_t *inst;
5529         boolean_t isall, isnone, rebound = B_FALSE;
5530 
5531         /* Validate fmri */
5532         isall = (strcmp(fmri, "all") == 0);
5533         isnone = (strcmp(fmri, "none") == 0);
5534 
5535         if (!isall && !isnone) {
5536                 if (fmri_canonify(fmri, (char **)&cfmri, B_FALSE) == EINVAL)
5537                         goto reject;
5538 
5539                 if (strcmp(cfmri, single_user_fmri) != 0 &&
5540                     strcmp(cfmri, multi_user_fmri) != 0 &&
5541                     strcmp(cfmri, multi_user_svr_fmri) != 0) {
5542                         startd_free((void *)cfmri, max_scf_fmri_size);
5543 reject:
5544                         log_framework(LOG_WARNING,
5545                             "Rejecting request for invalid milestone \"%s\".\n",
5546                             fmri);
5547                         return (EINVAL);
5548                 }
5549         }
5550 
5551         inst = safe_scf_instance_create(h);
5552 
5553         MUTEX_LOCK(&dgraph_lock);
5554 
5555         if (milestone == NULL) {
5556                 if (isall) {
5557                         log_framework(LOG_DEBUG,
5558                             "Milestone already set to all.\n");
5559                         ret = EALREADY;
5560                         goto out;
5561                 }
5562         } else if (milestone == MILESTONE_NONE) {
5563                 if (isnone) {
5564                         log_framework(LOG_DEBUG,
5565                             "Milestone already set to none.\n");
5566                         ret = EALREADY;
5567                         goto out;
5568                 }
5569         } else {
5570                 if (!isall && !isnone &&
5571                     strcmp(cfmri, milestone->gv_name) == 0) {
5572                         log_framework(LOG_DEBUG,
5573                             "Milestone already set to %s.\n", cfmri);
5574                         ret = EALREADY;
5575                         goto out;
5576                 }
5577         }
5578 
5579         if (!isall && !isnone) {
5580                 nm = vertex_get_by_name(cfmri);
5581                 if (nm == NULL || !(nm->gv_flags & GV_CONFIGURED)) {
5582                         log_framework(LOG_WARNING, "Cannot set milestone to %s "
5583                             "because no such service exists.\n", cfmri);
5584                         ret = ENOENT;
5585                         goto out;
5586                 }
5587         }
5588 
5589         log_framework(LOG_DEBUG, "Changing milestone to %s.\n", fmri);
5590 
5591         /*
5592          * Set milestone, removing the old one if this was the last reference.
5593          */
5594         if (milestone > MILESTONE_NONE)
5595                 (void) vertex_unref(milestone);
5596 
5597         if (isall)
5598                 milestone = NULL;
5599         else if (isnone)
5600                 milestone = MILESTONE_NONE;
5601         else {
5602                 milestone = nm;
5603                 /* milestone should count as a reference */
5604                 vertex_ref(milestone);
5605         }
5606 
5607         /* Clear all GV_INSUBGRAPH bits. */
5608         for (v = uu_list_first(dgraph); v != NULL; v = uu_list_next(dgraph, v))
5609                 v->gv_flags &= ~GV_INSUBGRAPH;
5610 
5611         if (!isall && !isnone) {
5612                 /* Set GV_INSUBGRAPH for milestone & descendents. */
5613                 milestone->gv_flags |= GV_INSUBGRAPH;
5614 
5615                 r = uu_list_walk(milestone->gv_dependencies,
5616                     (uu_walk_fn_t *)mark_subgraph, NULL, 0);
5617                 assert(r == 0);
5618         }
5619 
5620         /* Un-override services in the subgraph & override-disable the rest. */
5621         if (norepository)
5622                 goto out;
5623 
5624         non_subgraph_svcs = 0;
5625         for (v = uu_list_first(dgraph);
5626             v != NULL;
5627             v = uu_list_next(dgraph, v)) {
5628                 if (v->gv_type != GVT_INST ||
5629                     (v->gv_flags & GV_CONFIGURED) == 0)
5630                         continue;
5631 
5632 again:
5633                 r = scf_handle_decode_fmri(h, v->gv_name, NULL, NULL, inst,
5634                     NULL, NULL, SCF_DECODE_FMRI_EXACT);
5635                 if (r != 0) {
5636                         switch (scf_error()) {
5637                         case SCF_ERROR_CONNECTION_BROKEN:
5638                         default:
5639                                 libscf_handle_rebind(h);
5640                                 rebound = B_TRUE;
5641                                 goto again;
5642 
5643                         case SCF_ERROR_NOT_FOUND:
5644                                 continue;
5645 
5646                         case SCF_ERROR_HANDLE_MISMATCH:
5647                         case SCF_ERROR_INVALID_ARGUMENT:
5648                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5649                         case SCF_ERROR_NOT_BOUND:
5650                                 bad_error("scf_handle_decode_fmri",
5651                                     scf_error());
5652                         }
5653                 }
5654 
5655                 if (isall || (v->gv_flags & GV_INSUBGRAPH)) {
5656                         r = libscf_delete_enable_ovr(inst);
5657                         fs = "libscf_delete_enable_ovr";
5658                 } else {
5659                         assert(isnone || (v->gv_flags & GV_INSUBGRAPH) == 0);
5660 
5661                         /*
5662                          * Services which are up need to come down before
5663                          * we're done, but we can only disable the leaves
5664                          * here.
5665                          */
5666 
5667                         if (up_state(v->gv_state))
5668                                 ++non_subgraph_svcs;
5669 
5670                         /* If it's already disabled, don't bother. */
5671                         if ((v->gv_flags & GV_ENABLED) == 0)
5672                                 continue;
5673 
5674                         if (!is_nonsubgraph_leaf(v))
5675                                 continue;
5676 
5677                         r = libscf_set_enable_ovr(inst, 0);
5678                         fs = "libscf_set_enable_ovr";
5679                 }
5680                 switch (r) {
5681                 case 0:
5682                 case ECANCELED:
5683                         break;
5684 
5685                 case ECONNABORTED:
5686                         libscf_handle_rebind(h);
5687                         rebound = B_TRUE;
5688                         goto again;
5689 
5690                 case EPERM:
5691                 case EROFS:
5692                         log_error(LOG_WARNING,
5693                             "Could not set %s/%s for %s: %s.\n",
5694                             SCF_PG_GENERAL_OVR, SCF_PROPERTY_ENABLED,
5695                             v->gv_name, strerror(r));
5696                         break;
5697 
5698                 default:
5699                         bad_error(fs, r);
5700                 }
5701         }
5702 
5703         if (halting != -1) {
5704                 if (non_subgraph_svcs > 1)
5705                         uu_warn("%d system services are now being stopped.\n",
5706                             non_subgraph_svcs);
5707                 else if (non_subgraph_svcs == 1)
5708                         uu_warn("One system service is now being stopped.\n");
5709                 else if (non_subgraph_svcs == 0)
5710                         do_uadmin();
5711         }
5712 
5713         ret = rebound ? ECONNRESET : 0;
5714 
5715 out:
5716         MUTEX_UNLOCK(&dgraph_lock);
5717         if (!isall && !isnone)
5718                 startd_free((void *)cfmri, max_scf_fmri_size);
5719         scf_instance_destroy(inst);
5720         return (ret);
5721 }
5722 
5723 
5724 /*
5725  * Returns 0, ECONNABORTED, or EINVAL.
5726  */
5727 static int
5728 handle_graph_update_event(scf_handle_t *h, graph_protocol_event_t *e)
5729 {
5730         int r;
5731 
5732         switch (e->gpe_type) {
5733         case GRAPH_UPDATE_RELOAD_GRAPH:
5734                 log_error(LOG_WARNING,
5735                     "graph_event: reload graph unimplemented\n");
5736                 break;
5737 
5738         case GRAPH_UPDATE_STATE_CHANGE: {
5739                 protocol_states_t *states = e->gpe_data;
5740 
5741                 switch (r = dgraph_set_instance_state(h, e->gpe_inst, states)) {
5742                 case 0:
5743                 case ENOENT:
5744                         break;
5745 
5746                 case ECONNABORTED:
5747                         return (ECONNABORTED);
5748 
5749                 case EINVAL:
5750                 default:
5751 #ifndef NDEBUG
5752                         (void) fprintf(stderr, "dgraph_set_instance_state() "
5753                             "failed with unexpected error %d at %s:%d.\n", r,
5754                             __FILE__, __LINE__);
5755 #endif
5756                         abort();
5757                 }
5758 
5759                 startd_free(states, sizeof (protocol_states_t));
5760                 break;
5761         }
5762 
5763         default:
5764                 log_error(LOG_WARNING,
5765                     "graph_event_loop received an unknown event: %d\n",
5766                     e->gpe_type);
5767                 break;
5768         }
5769 
5770         return (0);
5771 }
5772 
5773 /*
5774  * graph_event_thread()
5775  *    Wait for state changes from the restarters.
5776  */
5777 /*ARGSUSED*/
5778 void *
5779 graph_event_thread(void *unused)
5780 {
5781         scf_handle_t *h;
5782         int err;
5783 
5784         h = libscf_handle_create_bound_loop();
5785 
5786         /*CONSTCOND*/
5787         while (1) {
5788                 graph_protocol_event_t *e;
5789 
5790                 MUTEX_LOCK(&gu->gu_lock);
5791 
5792                 while (gu->gu_wakeup == 0)
5793                         (void) pthread_cond_wait(&gu->gu_cv, &gu->gu_lock);
5794 
5795                 gu->gu_wakeup = 0;
5796 
5797                 while ((e = graph_event_dequeue()) != NULL) {
5798                         MUTEX_LOCK(&e->gpe_lock);
5799                         MUTEX_UNLOCK(&gu->gu_lock);
5800 
5801                         while ((err = handle_graph_update_event(h, e)) ==
5802                             ECONNABORTED)
5803                                 libscf_handle_rebind(h);
5804 
5805                         if (err == 0)
5806                                 graph_event_release(e);
5807                         else
5808                                 graph_event_requeue(e);
5809 
5810                         MUTEX_LOCK(&gu->gu_lock);
5811                 }
5812 
5813                 MUTEX_UNLOCK(&gu->gu_lock);
5814         }
5815 
5816         /*
5817          * Unreachable for now -- there's currently no graceful cleanup
5818          * called on exit().
5819          */
5820         MUTEX_UNLOCK(&gu->gu_lock);
5821         scf_handle_destroy(h);
5822         return (NULL);
5823 }
5824 
5825 static void
5826 set_initial_milestone(scf_handle_t *h)
5827 {
5828         scf_instance_t *inst;
5829         char *fmri, *cfmri;
5830         size_t sz;
5831         int r;
5832 
5833         inst = safe_scf_instance_create(h);
5834         fmri = startd_alloc(max_scf_fmri_size);
5835 
5836         /*
5837          * If -m milestone= was specified, we want to set options_ovr/milestone
5838          * to it.  Otherwise we want to read what the milestone should be set
5839          * to.  Either way we need our inst.
5840          */
5841 get_self:
5842         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL, inst,
5843             NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5844                 switch (scf_error()) {
5845                 case SCF_ERROR_CONNECTION_BROKEN:
5846                         libscf_handle_rebind(h);
5847                         goto get_self;
5848 
5849                 case SCF_ERROR_NOT_FOUND:
5850                         if (st->st_subgraph != NULL &&
5851                             st->st_subgraph[0] != '\0') {
5852                                 sz = strlcpy(fmri, st->st_subgraph,
5853                                     max_scf_fmri_size);
5854                                 assert(sz < max_scf_fmri_size);
5855                         } else {
5856                                 fmri[0] = '\0';
5857                         }
5858                         break;
5859 
5860                 case SCF_ERROR_INVALID_ARGUMENT:
5861                 case SCF_ERROR_CONSTRAINT_VIOLATED:
5862                 case SCF_ERROR_HANDLE_MISMATCH:
5863                 default:
5864                         bad_error("scf_handle_decode_fmri", scf_error());
5865                 }
5866         } else {
5867                 if (st->st_subgraph != NULL && st->st_subgraph[0] != '\0') {
5868                         scf_propertygroup_t *pg;
5869 
5870                         pg = safe_scf_pg_create(h);
5871 
5872                         sz = strlcpy(fmri, st->st_subgraph, max_scf_fmri_size);
5873                         assert(sz < max_scf_fmri_size);
5874 
5875                         r = libscf_inst_get_or_add_pg(inst, SCF_PG_OPTIONS_OVR,
5876                             SCF_PG_OPTIONS_OVR_TYPE, SCF_PG_OPTIONS_OVR_FLAGS,
5877                             pg);
5878                         switch (r) {
5879                         case 0:
5880                                 break;
5881 
5882                         case ECONNABORTED:
5883                                 libscf_handle_rebind(h);
5884                                 goto get_self;
5885 
5886                         case EPERM:
5887                         case EACCES:
5888                         case EROFS:
5889                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5890                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5891                                     SCF_PROPERTY_MILESTONE, strerror(r));
5892                                 /* FALLTHROUGH */
5893 
5894                         case ECANCELED:
5895                                 sz = strlcpy(fmri, st->st_subgraph,
5896                                     max_scf_fmri_size);
5897                                 assert(sz < max_scf_fmri_size);
5898                                 break;
5899 
5900                         default:
5901                                 bad_error("libscf_inst_get_or_add_pg", r);
5902                         }
5903 
5904                         r = libscf_clear_runlevel(pg, fmri);
5905                         switch (r) {
5906                         case 0:
5907                                 break;
5908 
5909                         case ECONNABORTED:
5910                                 libscf_handle_rebind(h);
5911                                 goto get_self;
5912 
5913                         case EPERM:
5914                         case EACCES:
5915                         case EROFS:
5916                                 log_error(LOG_WARNING, "Could not set %s/%s: "
5917                                     "%s.\n", SCF_PG_OPTIONS_OVR,
5918                                     SCF_PROPERTY_MILESTONE, strerror(r));
5919                                 /* FALLTHROUGH */
5920 
5921                         case ECANCELED:
5922                                 sz = strlcpy(fmri, st->st_subgraph,
5923                                     max_scf_fmri_size);
5924                                 assert(sz < max_scf_fmri_size);
5925                                 break;
5926 
5927                         default:
5928                                 bad_error("libscf_clear_runlevel", r);
5929                         }
5930 
5931                         scf_pg_destroy(pg);
5932                 } else {
5933                         scf_property_t *prop;
5934                         scf_value_t *val;
5935 
5936                         prop = safe_scf_property_create(h);
5937                         val = safe_scf_value_create(h);
5938 
5939                         r = libscf_get_milestone(inst, prop, val, fmri,
5940                             max_scf_fmri_size);
5941                         switch (r) {
5942                         case 0:
5943                                 break;
5944 
5945                         case ECONNABORTED:
5946                                 libscf_handle_rebind(h);
5947                                 goto get_self;
5948 
5949                         case EINVAL:
5950                                 log_error(LOG_WARNING, "Milestone property is "
5951                                     "misconfigured.  Defaulting to \"all\".\n");
5952                                 /* FALLTHROUGH */
5953 
5954                         case ECANCELED:
5955                         case ENOENT:
5956                                 fmri[0] = '\0';
5957                                 break;
5958 
5959                         default:
5960                                 bad_error("libscf_get_milestone", r);
5961                         }
5962 
5963                         scf_value_destroy(val);
5964                         scf_property_destroy(prop);
5965                 }
5966         }
5967 
5968         if (fmri[0] == '\0' || strcmp(fmri, "all") == 0)
5969                 goto out;
5970 
5971         if (strcmp(fmri, "none") != 0) {
5972 retry:
5973                 if (scf_handle_decode_fmri(h, fmri, NULL, NULL, inst, NULL,
5974                     NULL, SCF_DECODE_FMRI_EXACT) != 0) {
5975                         switch (scf_error()) {
5976                         case SCF_ERROR_INVALID_ARGUMENT:
5977                                 log_error(LOG_WARNING,
5978                                     "Requested milestone \"%s\" is invalid.  "
5979                                     "Reverting to \"all\".\n", fmri);
5980                                 goto out;
5981 
5982                         case SCF_ERROR_CONSTRAINT_VIOLATED:
5983                                 log_error(LOG_WARNING, "Requested milestone "
5984                                     "\"%s\" does not specify an instance.  "
5985                                     "Reverting to \"all\".\n", fmri);
5986                                 goto out;
5987 
5988                         case SCF_ERROR_CONNECTION_BROKEN:
5989                                 libscf_handle_rebind(h);
5990                                 goto retry;
5991 
5992                         case SCF_ERROR_NOT_FOUND:
5993                                 log_error(LOG_WARNING, "Requested milestone "
5994                                     "\"%s\" not in repository.  Reverting to "
5995                                     "\"all\".\n", fmri);
5996                                 goto out;
5997 
5998                         case SCF_ERROR_HANDLE_MISMATCH:
5999                         default:
6000                                 bad_error("scf_handle_decode_fmri",
6001                                     scf_error());
6002                         }
6003                 }
6004 
6005                 r = fmri_canonify(fmri, &cfmri, B_FALSE);
6006                 assert(r == 0);
6007 
6008                 r = dgraph_add_instance(cfmri, inst, B_TRUE);
6009                 startd_free(cfmri, max_scf_fmri_size);
6010                 switch (r) {
6011                 case 0:
6012                         break;
6013 
6014                 case ECONNABORTED:
6015                         goto retry;
6016 
6017                 case EINVAL:
6018                         log_error(LOG_WARNING,
6019                             "Requested milestone \"%s\" is invalid.  "
6020                             "Reverting to \"all\".\n", fmri);
6021                         goto out;
6022 
6023                 case ECANCELED:
6024                         log_error(LOG_WARNING,
6025                             "Requested milestone \"%s\" not "
6026                             "in repository.  Reverting to \"all\".\n",
6027                             fmri);
6028                         goto out;
6029 
6030                 case EEXIST:
6031                 default:
6032                         bad_error("dgraph_add_instance", r);
6033                 }
6034         }
6035 
6036         log_console(LOG_INFO, "Booting to milestone \"%s\".\n", fmri);
6037 
6038         r = dgraph_set_milestone(fmri, h, B_FALSE);
6039         switch (r) {
6040         case 0:
6041         case ECONNRESET:
6042         case EALREADY:
6043                 break;
6044 
6045         case EINVAL:
6046         case ENOENT:
6047         default:
6048                 bad_error("dgraph_set_milestone", r);
6049         }
6050 
6051 out:
6052         startd_free(fmri, max_scf_fmri_size);
6053         scf_instance_destroy(inst);
6054 }
6055 
6056 void
6057 set_restart_milestone(scf_handle_t *h)
6058 {
6059         scf_instance_t *inst;
6060         scf_property_t *prop;
6061         scf_value_t *val;
6062         char *fmri;
6063         int r;
6064 
6065         inst = safe_scf_instance_create(h);
6066 
6067 get_self:
6068         if (scf_handle_decode_fmri(h, SCF_SERVICE_STARTD, NULL, NULL,
6069             inst, NULL, NULL, SCF_DECODE_FMRI_EXACT) != 0) {
6070                 switch (scf_error()) {
6071                 case SCF_ERROR_CONNECTION_BROKEN:
6072                         libscf_handle_rebind(h);
6073                         goto get_self;
6074 
6075                 case SCF_ERROR_NOT_FOUND:
6076                         break;
6077 
6078                 case SCF_ERROR_INVALID_ARGUMENT:
6079                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6080                 case SCF_ERROR_HANDLE_MISMATCH:
6081                 default:
6082                         bad_error("scf_handle_decode_fmri", scf_error());
6083                 }
6084 
6085                 scf_instance_destroy(inst);
6086                 return;
6087         }
6088 
6089         prop = safe_scf_property_create(h);
6090         val = safe_scf_value_create(h);
6091         fmri = startd_alloc(max_scf_fmri_size);
6092 
6093         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6094         switch (r) {
6095         case 0:
6096                 break;
6097 
6098         case ECONNABORTED:
6099                 libscf_handle_rebind(h);
6100                 goto get_self;
6101 
6102         case ECANCELED:
6103         case ENOENT:
6104         case EINVAL:
6105                 goto out;
6106 
6107         default:
6108                 bad_error("libscf_get_milestone", r);
6109         }
6110 
6111         r = dgraph_set_milestone(fmri, h, B_TRUE);
6112         switch (r) {
6113         case 0:
6114         case ECONNRESET:
6115         case EALREADY:
6116         case EINVAL:
6117         case ENOENT:
6118                 break;
6119 
6120         default:
6121                 bad_error("dgraph_set_milestone", r);
6122         }
6123 
6124 out:
6125         startd_free(fmri, max_scf_fmri_size);
6126         scf_value_destroy(val);
6127         scf_property_destroy(prop);
6128         scf_instance_destroy(inst);
6129 }
6130 
6131 /*
6132  * void *graph_thread(void *)
6133  *
6134  * Graph management thread.
6135  */
6136 /*ARGSUSED*/
6137 void *
6138 graph_thread(void *arg)
6139 {
6140         scf_handle_t *h;
6141         int err;
6142 
6143         h = libscf_handle_create_bound_loop();
6144 
6145         if (st->st_initial)
6146                 set_initial_milestone(h);
6147 
6148         MUTEX_LOCK(&dgraph_lock);
6149         initial_milestone_set = B_TRUE;
6150         err = pthread_cond_broadcast(&initial_milestone_cv);
6151         assert(err == 0);
6152         MUTEX_UNLOCK(&dgraph_lock);
6153 
6154         libscf_populate_graph(h);
6155 
6156         if (!st->st_initial)
6157                 set_restart_milestone(h);
6158 
6159         MUTEX_LOCK(&st->st_load_lock);
6160         st->st_load_complete = 1;
6161         (void) pthread_cond_broadcast(&st->st_load_cv);
6162         MUTEX_UNLOCK(&st->st_load_lock);
6163 
6164         MUTEX_LOCK(&dgraph_lock);
6165         /*
6166          * Now that we've set st_load_complete we need to check can_come_up()
6167          * since if we booted to a milestone, then there won't be any more
6168          * state updates.
6169          */
6170         if (!go_single_user_mode && !go_to_level1 &&
6171             halting == -1) {
6172                 if (!sulogin_thread_running && !can_come_up()) {
6173                         (void) startd_thread_create(sulogin_thread, NULL);
6174                         sulogin_thread_running = B_TRUE;
6175                 }
6176         }
6177         MUTEX_UNLOCK(&dgraph_lock);
6178 
6179         (void) pthread_mutex_lock(&gu->gu_freeze_lock);
6180 
6181         /*CONSTCOND*/
6182         while (1) {
6183                 (void) pthread_cond_wait(&gu->gu_freeze_cv,
6184                     &gu->gu_freeze_lock);
6185         }
6186 
6187         /*
6188          * Unreachable for now -- there's currently no graceful cleanup
6189          * called on exit().
6190          */
6191         (void) pthread_mutex_unlock(&gu->gu_freeze_lock);
6192         scf_handle_destroy(h);
6193 
6194         return (NULL);
6195 }
6196 
6197 
6198 /*
6199  * int next_action()
6200  *   Given an array of timestamps 'a' with 'num' elements, find the
6201  *   lowest non-zero timestamp and return its index. If there are no
6202  *   non-zero elements, return -1.
6203  */
6204 static int
6205 next_action(hrtime_t *a, int num)
6206 {
6207         hrtime_t t = 0;
6208         int i = 0, smallest = -1;
6209 
6210         for (i = 0; i < num; i++) {
6211                 if (t == 0) {
6212                         t = a[i];
6213                         smallest = i;
6214                 } else if (a[i] != 0 && a[i] < t) {
6215                         t = a[i];
6216                         smallest = i;
6217                 }
6218         }
6219 
6220         if (t == 0)
6221                 return (-1);
6222         else
6223                 return (smallest);
6224 }
6225 
6226 /*
6227  * void process_actions()
6228  *   Process actions requested by the administrator. Possibilities include:
6229  *   refresh, restart, maintenance mode off, maintenance mode on,
6230  *   maintenance mode immediate, and degraded.
6231  *
6232  *   The set of pending actions is represented in the repository as a
6233  *   per-instance property group, with each action being a single property
6234  *   in that group.  This property group is converted to an array, with each
6235  *   action type having an array slot.  The actions in the array at the
6236  *   time process_actions() is called are acted on in the order of the
6237  *   timestamp (which is the value stored in the slot).  A value of zero
6238  *   indicates that there is no pending action of the type associated with
6239  *   a particular slot.
6240  *
6241  *   Sending an action event multiple times before the restarter has a
6242  *   chance to process that action will force it to be run at the last
6243  *   timestamp where it appears in the ordering.
6244  *
6245  *   Turning maintenance mode on trumps all other actions.
6246  *
6247  *   Returns 0 or ECONNABORTED.
6248  */
6249 static int
6250 process_actions(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst)
6251 {
6252         scf_property_t *prop = NULL;
6253         scf_value_t *val = NULL;
6254         scf_type_t type;
6255         graph_vertex_t *vertex;
6256         admin_action_t a;
6257         int i, ret = 0, r;
6258         hrtime_t action_ts[NACTIONS];
6259         char *inst_name;
6260 
6261         r = libscf_instance_get_fmri(inst, &inst_name);
6262         switch (r) {
6263         case 0:
6264                 break;
6265 
6266         case ECONNABORTED:
6267                 return (ECONNABORTED);
6268 
6269         case ECANCELED:
6270                 return (0);
6271 
6272         default:
6273                 bad_error("libscf_instance_get_fmri", r);
6274         }
6275 
6276         MUTEX_LOCK(&dgraph_lock);
6277 
6278         vertex = vertex_get_by_name(inst_name);
6279         if (vertex == NULL) {
6280                 MUTEX_UNLOCK(&dgraph_lock);
6281                 log_framework(LOG_DEBUG, "%s: Can't find graph vertex. "
6282                     "The instance must have been removed.\n", inst_name);
6283                 startd_free(inst_name, max_scf_fmri_size);
6284                 return (0);
6285         }
6286 
6287         prop = safe_scf_property_create(h);
6288         val = safe_scf_value_create(h);
6289 
6290         for (i = 0; i < NACTIONS; i++) {
6291                 if (scf_pg_get_property(pg, admin_actions[i], prop) != 0) {
6292                         switch (scf_error()) {
6293                         case SCF_ERROR_CONNECTION_BROKEN:
6294                         default:
6295                                 ret = ECONNABORTED;
6296                                 goto out;
6297 
6298                         case SCF_ERROR_DELETED:
6299                                 goto out;
6300 
6301                         case SCF_ERROR_NOT_FOUND:
6302                                 action_ts[i] = 0;
6303                                 continue;
6304 
6305                         case SCF_ERROR_HANDLE_MISMATCH:
6306                         case SCF_ERROR_INVALID_ARGUMENT:
6307                         case SCF_ERROR_NOT_SET:
6308                                 bad_error("scf_pg_get_property", scf_error());
6309                         }
6310                 }
6311 
6312                 if (scf_property_type(prop, &type) != 0) {
6313                         switch (scf_error()) {
6314                         case SCF_ERROR_CONNECTION_BROKEN:
6315                         default:
6316                                 ret = ECONNABORTED;
6317                                 goto out;
6318 
6319                         case SCF_ERROR_DELETED:
6320                                 action_ts[i] = 0;
6321                                 continue;
6322 
6323                         case SCF_ERROR_NOT_SET:
6324                                 bad_error("scf_property_type", scf_error());
6325                         }
6326                 }
6327 
6328                 if (type != SCF_TYPE_INTEGER) {
6329                         action_ts[i] = 0;
6330                         continue;
6331                 }
6332 
6333                 if (scf_property_get_value(prop, val) != 0) {
6334                         switch (scf_error()) {
6335                         case SCF_ERROR_CONNECTION_BROKEN:
6336                         default:
6337                                 ret = ECONNABORTED;
6338                                 goto out;
6339 
6340                         case SCF_ERROR_DELETED:
6341                                 goto out;
6342 
6343                         case SCF_ERROR_NOT_FOUND:
6344                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6345                                 action_ts[i] = 0;
6346                                 continue;
6347 
6348                         case SCF_ERROR_NOT_SET:
6349                         case SCF_ERROR_PERMISSION_DENIED:
6350                                 bad_error("scf_property_get_value",
6351                                     scf_error());
6352                         }
6353                 }
6354 
6355                 r = scf_value_get_integer(val, &action_ts[i]);
6356                 assert(r == 0);
6357         }
6358 
6359         a = ADMIN_EVENT_MAINT_ON_IMMEDIATE;
6360         if (action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ||
6361             action_ts[ADMIN_EVENT_MAINT_ON]) {
6362                 a = action_ts[ADMIN_EVENT_MAINT_ON_IMMEDIATE] ?
6363                     ADMIN_EVENT_MAINT_ON_IMMEDIATE : ADMIN_EVENT_MAINT_ON;
6364 
6365                 vertex_send_event(vertex, admin_events[a]);
6366                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6367                 switch (r) {
6368                 case 0:
6369                 case EACCES:
6370                         break;
6371 
6372                 case ECONNABORTED:
6373                         ret = ECONNABORTED;
6374                         goto out;
6375 
6376                 case EPERM:
6377                         uu_die("Insufficient privilege.\n");
6378                         /* NOTREACHED */
6379 
6380                 default:
6381                         bad_error("libscf_unset_action", r);
6382                 }
6383         }
6384 
6385         while ((a = next_action(action_ts, NACTIONS)) != -1) {
6386                 log_framework(LOG_DEBUG,
6387                     "Graph: processing %s action for %s.\n", admin_actions[a],
6388                     inst_name);
6389 
6390                 if (a == ADMIN_EVENT_REFRESH) {
6391                         r = dgraph_refresh_instance(vertex, inst);
6392                         switch (r) {
6393                         case 0:
6394                         case ECANCELED:
6395                         case EINVAL:
6396                         case -1:
6397                                 break;
6398 
6399                         case ECONNABORTED:
6400                                 /* pg & inst are reset now, so just return. */
6401                                 ret = ECONNABORTED;
6402                                 goto out;
6403 
6404                         default:
6405                                 bad_error("dgraph_refresh_instance", r);
6406                         }
6407                 }
6408 
6409                 vertex_send_event(vertex, admin_events[a]);
6410 
6411                 r = libscf_unset_action(h, pg, a, action_ts[a]);
6412                 switch (r) {
6413                 case 0:
6414                 case EACCES:
6415                         break;
6416 
6417                 case ECONNABORTED:
6418                         ret = ECONNABORTED;
6419                         goto out;
6420 
6421                 case EPERM:
6422                         uu_die("Insufficient privilege.\n");
6423                         /* NOTREACHED */
6424 
6425                 default:
6426                         bad_error("libscf_unset_action", r);
6427                 }
6428 
6429                 action_ts[a] = 0;
6430         }
6431 
6432 out:
6433         MUTEX_UNLOCK(&dgraph_lock);
6434 
6435         scf_property_destroy(prop);
6436         scf_value_destroy(val);
6437         startd_free(inst_name, max_scf_fmri_size);
6438         return (ret);
6439 }
6440 
6441 /*
6442  * inst and pg_name are scratch space, and are unset on entry.
6443  * Returns
6444  *   0 - success
6445  *   ECONNRESET - success, but repository handle rebound
6446  *   ECONNABORTED - repository connection broken
6447  */
6448 static int
6449 process_pg_event(scf_handle_t *h, scf_propertygroup_t *pg, scf_instance_t *inst,
6450     char *pg_name)
6451 {
6452         int r;
6453         scf_property_t *prop;
6454         scf_value_t *val;
6455         char *fmri;
6456         boolean_t rebound = B_FALSE, rebind_inst = B_FALSE;
6457 
6458         if (scf_pg_get_name(pg, pg_name, max_scf_value_size) < 0) {
6459                 switch (scf_error()) {
6460                 case SCF_ERROR_CONNECTION_BROKEN:
6461                 default:
6462                         return (ECONNABORTED);
6463 
6464                 case SCF_ERROR_DELETED:
6465                         return (0);
6466 
6467                 case SCF_ERROR_NOT_SET:
6468                         bad_error("scf_pg_get_name", scf_error());
6469                 }
6470         }
6471 
6472         if (strcmp(pg_name, SCF_PG_GENERAL) == 0 ||
6473             strcmp(pg_name, SCF_PG_GENERAL_OVR) == 0) {
6474                 r = dgraph_update_general(pg);
6475                 switch (r) {
6476                 case 0:
6477                 case ENOTSUP:
6478                 case ECANCELED:
6479                         return (0);
6480 
6481                 case ECONNABORTED:
6482                         return (ECONNABORTED);
6483 
6484                 case -1:
6485                         /* Error should have been logged. */
6486                         return (0);
6487 
6488                 default:
6489                         bad_error("dgraph_update_general", r);
6490                 }
6491         } else if (strcmp(pg_name, SCF_PG_RESTARTER_ACTIONS) == 0) {
6492                 if (scf_pg_get_parent_instance(pg, inst) != 0) {
6493                         switch (scf_error()) {
6494                         case SCF_ERROR_CONNECTION_BROKEN:
6495                                 return (ECONNABORTED);
6496 
6497                         case SCF_ERROR_DELETED:
6498                         case SCF_ERROR_CONSTRAINT_VIOLATED:
6499                                 /* Ignore commands on services. */
6500                                 return (0);
6501 
6502                         case SCF_ERROR_NOT_BOUND:
6503                         case SCF_ERROR_HANDLE_MISMATCH:
6504                         case SCF_ERROR_NOT_SET:
6505                         default:
6506                                 bad_error("scf_pg_get_parent_instance",
6507                                     scf_error());
6508                         }
6509                 }
6510 
6511                 return (process_actions(h, pg, inst));
6512         }
6513 
6514         if (strcmp(pg_name, SCF_PG_OPTIONS) != 0 &&
6515             strcmp(pg_name, SCF_PG_OPTIONS_OVR) != 0)
6516                 return (0);
6517 
6518         /*
6519          * We only care about the options[_ovr] property groups of our own
6520          * instance, so get the fmri and compare.  Plus, once we know it's
6521          * correct, if the repository connection is broken we know exactly what
6522          * property group we were operating on, and can look it up again.
6523          */
6524         if (scf_pg_get_parent_instance(pg, inst) != 0) {
6525                 switch (scf_error()) {
6526                 case SCF_ERROR_CONNECTION_BROKEN:
6527                         return (ECONNABORTED);
6528 
6529                 case SCF_ERROR_DELETED:
6530                 case SCF_ERROR_CONSTRAINT_VIOLATED:
6531                         return (0);
6532 
6533                 case SCF_ERROR_HANDLE_MISMATCH:
6534                 case SCF_ERROR_NOT_BOUND:
6535                 case SCF_ERROR_NOT_SET:
6536                 default:
6537                         bad_error("scf_pg_get_parent_instance",
6538                             scf_error());
6539                 }
6540         }
6541 
6542         switch (r = libscf_instance_get_fmri(inst, &fmri)) {
6543         case 0:
6544                 break;
6545 
6546         case ECONNABORTED:
6547                 return (ECONNABORTED);
6548 
6549         case ECANCELED:
6550                 return (0);
6551 
6552         default:
6553                 bad_error("libscf_instance_get_fmri", r);
6554         }
6555 
6556         if (strcmp(fmri, SCF_SERVICE_STARTD) != 0) {
6557                 startd_free(fmri, max_scf_fmri_size);
6558                 return (0);
6559         }
6560 
6561         /*
6562          * update the information events flag
6563          */
6564         if (strcmp(pg_name, SCF_PG_OPTIONS) == 0)
6565                 info_events_all = libscf_get_info_events_all(pg);
6566 
6567         prop = safe_scf_property_create(h);
6568         val = safe_scf_value_create(h);
6569 
6570         if (strcmp(pg_name, SCF_PG_OPTIONS_OVR) == 0) {
6571                 /* See if we need to set the runlevel. */
6572                 /* CONSTCOND */
6573                 if (0) {
6574 rebind_pg:
6575                         libscf_handle_rebind(h);
6576                         rebound = B_TRUE;
6577 
6578                         r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6579                         switch (r) {
6580                         case 0:
6581                                 break;
6582 
6583                         case ECONNABORTED:
6584                                 goto rebind_pg;
6585 
6586                         case ENOENT:
6587                                 goto out;
6588 
6589                         case EINVAL:
6590                         case ENOTSUP:
6591                                 bad_error("libscf_lookup_instance", r);
6592                         }
6593 
6594                         if (scf_instance_get_pg(inst, pg_name, pg) != 0) {
6595                                 switch (scf_error()) {
6596                                 case SCF_ERROR_DELETED:
6597                                 case SCF_ERROR_NOT_FOUND:
6598                                         goto out;
6599 
6600                                 case SCF_ERROR_CONNECTION_BROKEN:
6601                                         goto rebind_pg;
6602 
6603                                 case SCF_ERROR_HANDLE_MISMATCH:
6604                                 case SCF_ERROR_NOT_BOUND:
6605                                 case SCF_ERROR_NOT_SET:
6606                                 case SCF_ERROR_INVALID_ARGUMENT:
6607                                 default:
6608                                         bad_error("scf_instance_get_pg",
6609                                             scf_error());
6610                                 }
6611                         }
6612                 }
6613 
6614                 if (scf_pg_get_property(pg, "runlevel", prop) == 0) {
6615                         r = dgraph_set_runlevel(pg, prop);
6616                         switch (r) {
6617                         case ECONNRESET:
6618                                 rebound = B_TRUE;
6619                                 rebind_inst = B_TRUE;
6620                                 /* FALLTHROUGH */
6621 
6622                         case 0:
6623                                 break;
6624 
6625                         case ECONNABORTED:
6626                                 goto rebind_pg;
6627 
6628                         case ECANCELED:
6629                                 goto out;
6630 
6631                         default:
6632                                 bad_error("dgraph_set_runlevel", r);
6633                         }
6634                 } else {
6635                         switch (scf_error()) {
6636                         case SCF_ERROR_CONNECTION_BROKEN:
6637                         default:
6638                                 goto rebind_pg;
6639 
6640                         case SCF_ERROR_DELETED:
6641                                 goto out;
6642 
6643                         case SCF_ERROR_NOT_FOUND:
6644                                 break;
6645 
6646                         case SCF_ERROR_INVALID_ARGUMENT:
6647                         case SCF_ERROR_HANDLE_MISMATCH:
6648                         case SCF_ERROR_NOT_BOUND:
6649                         case SCF_ERROR_NOT_SET:
6650                                 bad_error("scf_pg_get_property", scf_error());
6651                         }
6652                 }
6653         }
6654 
6655         if (rebind_inst) {
6656 lookup_inst:
6657                 r = libscf_lookup_instance(SCF_SERVICE_STARTD, inst);
6658                 switch (r) {
6659                 case 0:
6660                         break;
6661 
6662                 case ECONNABORTED:
6663                         libscf_handle_rebind(h);
6664                         rebound = B_TRUE;
6665                         goto lookup_inst;
6666 
6667                 case ENOENT:
6668                         goto out;
6669 
6670                 case EINVAL:
6671                 case ENOTSUP:
6672                         bad_error("libscf_lookup_instance", r);
6673                 }
6674         }
6675 
6676         r = libscf_get_milestone(inst, prop, val, fmri, max_scf_fmri_size);
6677         switch (r) {
6678         case 0:
6679                 break;
6680 
6681         case ECONNABORTED:
6682                 libscf_handle_rebind(h);
6683                 rebound = B_TRUE;
6684                 goto lookup_inst;
6685 
6686         case EINVAL:
6687                 log_error(LOG_NOTICE,
6688                     "%s/%s property of %s is misconfigured.\n", pg_name,
6689                     SCF_PROPERTY_MILESTONE, SCF_SERVICE_STARTD);
6690                 /* FALLTHROUGH */
6691 
6692         case ECANCELED:
6693         case ENOENT:
6694                 (void) strcpy(fmri, "all");
6695                 break;
6696 
6697         default:
6698                 bad_error("libscf_get_milestone", r);
6699         }
6700 
6701         r = dgraph_set_milestone(fmri, h, B_FALSE);
6702         switch (r) {
6703         case 0:
6704         case ECONNRESET:
6705         case EALREADY:
6706                 break;
6707 
6708         case EINVAL:
6709                 log_error(LOG_WARNING, "Milestone %s is invalid.\n", fmri);
6710                 break;
6711 
6712         case ENOENT:
6713                 log_error(LOG_WARNING, "Milestone %s does not exist.\n", fmri);
6714                 break;
6715 
6716         default:
6717                 bad_error("dgraph_set_milestone", r);
6718         }
6719 
6720 out:
6721         startd_free(fmri, max_scf_fmri_size);
6722         scf_value_destroy(val);
6723         scf_property_destroy(prop);
6724 
6725         return (rebound ? ECONNRESET : 0);
6726 }
6727 
6728 /*
6729  * process_delete() deletes an instance from the dgraph if 'fmri' is an
6730  * instance fmri or if 'fmri' matches the 'general' property group of an
6731  * instance (or the 'general/enabled' property).
6732  *
6733  * 'fmri' may be overwritten and cannot be trusted on return by the caller.
6734  */
6735 static void
6736 process_delete(char *fmri, scf_handle_t *h)
6737 {
6738         char *lfmri, *end_inst_fmri;
6739         const char *inst_name = NULL;
6740         const char *pg_name = NULL;
6741         const char *prop_name = NULL;
6742 
6743         lfmri = safe_strdup(fmri);
6744 
6745         /* Determine if the FMRI is a property group or instance */
6746         if (scf_parse_svc_fmri(lfmri, NULL, NULL, &inst_name, &pg_name,
6747             &prop_name) != SCF_SUCCESS) {
6748                 log_error(LOG_WARNING,
6749                     "Received invalid FMRI \"%s\" from repository server.\n",
6750                     fmri);
6751         } else if (inst_name != NULL && pg_name == NULL) {
6752                 (void) dgraph_remove_instance(fmri, h);
6753         } else if (inst_name != NULL && pg_name != NULL) {
6754                 /*
6755                  * If we're deleting the 'general' property group or
6756                  * 'general/enabled' property then the whole instance
6757                  * must be removed from the dgraph.
6758                  */
6759                 if (strcmp(pg_name, SCF_PG_GENERAL) != 0) {
6760                         free(lfmri);
6761                         return;
6762                 }
6763 
6764                 if (prop_name != NULL &&
6765                     strcmp(prop_name, SCF_PROPERTY_ENABLED) != 0) {
6766                         free(lfmri);
6767                         return;
6768                 }
6769 
6770                 /*
6771                  * Because the instance has already been deleted from the
6772                  * repository, we cannot use any scf_ functions to retrieve
6773                  * the instance FMRI however we can easily reconstruct it
6774                  * manually.
6775                  */
6776                 end_inst_fmri = strstr(fmri, SCF_FMRI_PROPERTYGRP_PREFIX);
6777                 if (end_inst_fmri == NULL)
6778                         bad_error("process_delete", 0);
6779 
6780                 end_inst_fmri[0] = '\0';
6781 
6782                 (void) dgraph_remove_instance(fmri, h);
6783         }
6784 
6785         free(lfmri);
6786 }
6787 
6788 /*ARGSUSED*/
6789 void *
6790 repository_event_thread(void *unused)
6791 {
6792         scf_handle_t *h;
6793         scf_propertygroup_t *pg;
6794         scf_instance_t *inst;
6795         char *fmri = startd_alloc(max_scf_fmri_size);
6796         char *pg_name = startd_alloc(max_scf_value_size);
6797         int r;
6798 
6799         h = libscf_handle_create_bound_loop();
6800 
6801         pg = safe_scf_pg_create(h);
6802         inst = safe_scf_instance_create(h);
6803 
6804 retry:
6805         if (_scf_notify_add_pgtype(h, SCF_GROUP_FRAMEWORK) != SCF_SUCCESS) {
6806                 if (scf_error() == SCF_ERROR_CONNECTION_BROKEN) {
6807                         libscf_handle_rebind(h);
6808                 } else {
6809                         log_error(LOG_WARNING,
6810                             "Couldn't set up repository notification "
6811                             "for property group type %s: %s\n",
6812                             SCF_GROUP_FRAMEWORK, scf_strerror(scf_error()));
6813 
6814                         (void) sleep(1);
6815                 }
6816 
6817                 goto retry;
6818         }
6819 
6820         /*CONSTCOND*/
6821         while (1) {
6822                 ssize_t res;
6823 
6824                 /* Note: fmri is only set on delete events. */
6825                 res = _scf_notify_wait(pg, fmri, max_scf_fmri_size);
6826                 if (res < 0) {
6827                         libscf_handle_rebind(h);
6828                         goto retry;
6829                 } else if (res == 0) {
6830                         /*
6831                          * property group modified.  inst and pg_name are
6832                          * pre-allocated scratch space.
6833                          */
6834                         if (scf_pg_update(pg) < 0) {
6835                                 switch (scf_error()) {
6836                                 case SCF_ERROR_DELETED:
6837                                         continue;
6838 
6839                                 case SCF_ERROR_CONNECTION_BROKEN:
6840                                         log_error(LOG_WARNING,
6841                                             "Lost repository event due to "
6842                                             "disconnection.\n");
6843                                         libscf_handle_rebind(h);
6844                                         goto retry;
6845 
6846                                 case SCF_ERROR_NOT_BOUND:
6847                                 case SCF_ERROR_NOT_SET:
6848                                 default:
6849                                         bad_error("scf_pg_update", scf_error());
6850                                 }
6851                         }
6852 
6853                         r = process_pg_event(h, pg, inst, pg_name);
6854                         switch (r) {
6855                         case 0:
6856                                 break;
6857 
6858                         case ECONNABORTED:
6859                                 log_error(LOG_WARNING, "Lost repository event "
6860                                     "due to disconnection.\n");
6861                                 libscf_handle_rebind(h);
6862                                 /* FALLTHROUGH */
6863 
6864                         case ECONNRESET:
6865                                 goto retry;
6866 
6867                         default:
6868                                 bad_error("process_pg_event", r);
6869                         }
6870                 } else {
6871                         /*
6872                          * Service, instance, or pg deleted.
6873                          * Don't trust fmri on return.
6874                          */
6875                         process_delete(fmri, h);
6876                 }
6877         }
6878 
6879         /*NOTREACHED*/
6880         return (NULL);
6881 }
6882 
6883 void
6884 graph_engine_start()
6885 {
6886         int err;
6887 
6888         (void) startd_thread_create(graph_thread, NULL);
6889 
6890         MUTEX_LOCK(&dgraph_lock);
6891         while (!initial_milestone_set) {
6892                 err = pthread_cond_wait(&initial_milestone_cv, &dgraph_lock);
6893                 assert(err == 0);
6894         }
6895         MUTEX_UNLOCK(&dgraph_lock);
6896 
6897         (void) startd_thread_create(repository_event_thread, NULL);
6898         (void) startd_thread_create(graph_event_thread, NULL);
6899 }