Print this page
7267 SMF is fast and loose with optional dependencies
Reviewed by: Dan McDonald <danmcd@omniti.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Albert Lee <trisk@omniti.com>


   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2015, Syneto S.R.L. All rights reserved.

  25  */
  26 
  27 /*
  28  * graph.c - master restarter graph engine
  29  *
  30  *   The graph engine keeps a dependency graph of all service instances on the
  31  *   system, as recorded in the repository.  It decides when services should
  32  *   be brought up or down based on service states and dependencies and sends
  33  *   commands to restarters to effect any changes.  It also executes
  34  *   administrator commands sent by svcadm via the repository.
  35  *
  36  *   The graph is stored in uu_list_t *dgraph and its vertices are
  37  *   graph_vertex_t's, each of which has a name and an integer id unique to
  38  *   its name (see dict.c).  A vertex's type attribute designates the type
  39  *   of object it represents: GVT_INST for service instances, GVT_SVC for
  40  *   service objects (since service instances may depend on another service,
  41  *   rather than service instance), GVT_FILE for files (which services may
  42  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
  43  *   vertices are necessary because dependency lists may have particular
  44  *   grouping types (require any, require all, optional, or exclude) and


 190  *     access to stn_global
 191  */
 192 int32_t stn_global;
 193 /*
 194  * info_events_all holds a flag to override notification parameters and send
 195  * Information events for all state transitions.
 196  * same about the need of a mutex here.
 197  */
 198 int info_events_all;
 199 
 200 /*
 201  * Services in these states are not considered 'down' by the
 202  * milestone/shutdown code.
 203  */
 204 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
 205         (state) == RESTARTER_STATE_DEGRADED || \
 206         (state) == RESTARTER_STATE_OFFLINE)
 207 
 208 #define is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
 209         ((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \
 210         (v->gv_depgroup == DEPGRP_OPTIONAL_ALL) || \
 211         (v->gv_restart < RERR_RESTART)))
 212 
 213 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
 214 static uu_list_t *dgraph;
 215 static pthread_mutex_t dgraph_lock;
 216 
 217 /*
 218  * milestone indicates the current subgraph.  When NULL, it is the entire
 219  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
 220  * services on which the target vertex depends.
 221  */
 222 static graph_vertex_t *milestone = NULL;
 223 static boolean_t initial_milestone_set = B_FALSE;
 224 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
 225 
 226 /* protected by dgraph_lock */
 227 static boolean_t sulogin_thread_running = B_FALSE;
 228 static boolean_t sulogin_running = B_FALSE;
 229 static boolean_t console_login_ready = B_FALSE;
 230 


1311  */
1312 static int
1313 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1314 {
1315         graph_edge_t *edge;
1316         graph_vertex_t *v;
1317         boolean_t any_qualified;
1318         boolean_t any_unsatisfied;
1319         int i;
1320 
1321         any_qualified = B_FALSE;
1322         any_unsatisfied = B_FALSE;
1323 
1324         for (edge = uu_list_first(groupv->gv_dependencies);
1325             edge != NULL;
1326             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1327                 v = edge->ge_vertex;
1328 
1329                 switch (v->gv_type) {
1330                 case GVT_INST:
1331                         /* Skip missing or disabled instances */
1332                         if ((v->gv_flags & (GV_CONFIGURED | GV_ENABLED)) !=
1333                             (GV_CONFIGURED | GV_ENABLED))
1334                                 continue;
1335 
1336                         if (v->gv_state == RESTARTER_STATE_MAINT)
1337                                 continue;
1338 
1339                         if (v->gv_flags & GV_TOOFFLINE)
1340                                 continue;
1341 
1342                         any_qualified = B_TRUE;
1343                         if (v->gv_state == RESTARTER_STATE_OFFLINE) {
1344                                 /*
1345                                  * For offline dependencies, treat unsatisfiable
1346                                  * as satisfied.
1347                                  */
1348                                 i = dependency_satisfied(v, B_TRUE);
1349                                 if (i == -1)
1350                                         i = 1;
1351                         } else if (v->gv_state == RESTARTER_STATE_DISABLED) {
1352                                 /*
1353                                  * The service is enabled, but hasn't
1354                                  * transitioned out of disabled yet.  Treat it
1355                                  * as unsatisfied (not unsatisfiable).
1356                                  */
1357                                 i = 0;
1358                         } else {
1359                                 i = dependency_satisfied(v, satbility);
1360                         }
1361                         break;
1362 
1363                 case GVT_FILE:
1364                         any_qualified = B_TRUE;
1365                         i = dependency_satisfied(v, satbility);
1366 
1367                         break;
1368 
1369                 case GVT_SVC: {
1370                         boolean_t svc_any_qualified;
1371                         boolean_t svc_satisfied;
1372                         boolean_t svc_satisfiable;
1373                         graph_vertex_t *v2;
1374                         graph_edge_t *e2;
1375 
1376                         svc_any_qualified = B_FALSE;
1377                         svc_satisfied = B_FALSE;
1378                         svc_satisfiable = B_FALSE;
1379 
1380                         for (e2 = uu_list_first(v->gv_dependencies);
1381                             e2 != NULL;
1382                             e2 = uu_list_next(v->gv_dependencies, e2)) {
1383                                 v2 = e2->ge_vertex;
1384                                 assert(v2->gv_type == GVT_INST);
1385 
1386                                 if ((v2->gv_flags &
1387                                     (GV_CONFIGURED | GV_ENABLED)) !=
1388                                     (GV_CONFIGURED | GV_ENABLED))
1389                                         continue;
1390 
1391                                 if (v2->gv_state == RESTARTER_STATE_MAINT)
1392                                         continue;
1393 
1394                                 if (v2->gv_flags & GV_TOOFFLINE)
1395                                         continue;
1396 
1397                                 svc_any_qualified = B_TRUE;
1398 
1399                                 if (v2->gv_state == RESTARTER_STATE_OFFLINE) {
1400                                         /*
1401                                          * For offline dependencies, treat
1402                                          * unsatisfiable as satisfied.
1403                                          */
1404                                         i = dependency_satisfied(v2, B_TRUE);
1405                                         if (i == -1)
1406                                                 i = 1;
1407                                 } else if (v2->gv_state ==
1408                                     RESTARTER_STATE_DISABLED) {
1409                                         i = 0;
1410                                 } else {
1411                                         i = dependency_satisfied(v2, satbility);
1412                                 }
1413 
1414                                 if (i == 1) {
1415                                         svc_satisfied = B_TRUE;
1416                                         break;
1417                                 }
1418                                 if (i == 0)
1419                                         svc_satisfiable = B_TRUE;
1420                         }
1421 
1422                         if (!svc_any_qualified)
1423                                 continue;
1424                         any_qualified = B_TRUE;
1425                         if (svc_satisfied) {
1426                                 i = 1;
1427                         } else if (svc_satisfiable) {
1428                                 i = 0;
1429                         } else {
1430                                 i = -1;
1431                         }
1432                         break;
1433                 }
1434 
1435                 case GVT_GROUP:
1436                 default:
1437 #ifndef NDEBUG
1438                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1439                             __LINE__, v->gv_type);
1440 #endif
1441                         abort();
1442                 }
1443 
1444                 if (i == 1)
1445                         continue;
1446 
1447                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1448                     "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1449                     v->gv_name, i == 0 ? "ed" : "able");
1450 
1451                 if (!satbility)


1591  * Decide whether v can satisfy a dependency.  v can either be a child of
1592  * a group vertex, or of an instance vertex.
1593  */
1594 static int
1595 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1596 {
1597         switch (v->gv_type) {
1598         case GVT_INST:
1599                 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1600                         if (v->gv_flags & GV_DEATHROW) {
1601                                 /*
1602                                  * A dependency on an instance with GV_DEATHROW
1603                                  * flag is always considered as satisfied.
1604                                  */
1605                                 return (1);
1606                         }
1607                         return (-1);
1608                 }
1609 
1610                 /*
1611                  * Any vertex with the GV_TOOFFLINE flag set is guaranteed
1612                  * to have its dependencies unsatisfiable.

1613                  */
1614                 if (v->gv_flags & GV_TOOFFLINE)
1615                         return (-1);


1616 
1617                 switch (v->gv_state) {
1618                 case RESTARTER_STATE_ONLINE:
1619                 case RESTARTER_STATE_DEGRADED:
1620                         return (1);
1621 
1622                 case RESTARTER_STATE_OFFLINE:
1623                         if (!satbility)
1624                                 return (0);
1625                         return (instance_satisfied(v, satbility) != -1 ?
1626                             0 : -1);
1627 
1628                 case RESTARTER_STATE_DISABLED:
1629                 case RESTARTER_STATE_MAINT:
1630                         return (-1);
1631 
1632                 case RESTARTER_STATE_UNINIT:
1633                         return (0);
1634 
1635                 default:


1711  *      |  A  |--------------->|  B  |-------------->|  C  |
1712  *      +-----+                +-----+               +-----+
1713  *
1714  *                                              offline -> maintenance
1715  *
1716  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1717  * an optional dependency, what was previously an unsatisfiable situation is now
1718  * satisfied (B will never come online, even though its state hasn't changed).
1719  *
1720  * Note that it's not necessary to continue examining dependents after reaching
1721  * an optional_all dependency.  It's not possible for an optional_all dependency
1722  * to change satisfiability without also coming online, in which case we get a
1723  * start event and propagation continues naturally.  However, it does no harm to
1724  * continue propagating satisfiability (as it is a relatively rare event), and
1725  * keeps the walker code simple and generic.
1726  */
1727 /*ARGSUSED*/
1728 static int
1729 satbility_cb(graph_vertex_t *v, void *arg)
1730 {



1731         if (v->gv_type == GVT_INST)
1732                 graph_start_if_satisfied(v);
1733 
1734         return (UU_WALK_NEXT);
1735 }
1736 
1737 static void
1738 propagate_satbility(graph_vertex_t *v)
1739 {
1740         graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1741 }
1742 
1743 static void propagate_stop(graph_vertex_t *, void *);
1744 
1745 /* ARGSUSED */








1746 static void
1747 propagate_start(graph_vertex_t *v, void *arg)
1748 {





1749         switch (v->gv_type) {
1750         case GVT_INST:







1751                 graph_start_if_satisfied(v);

1752                 break;
1753 
1754         case GVT_GROUP:
1755                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1756                         graph_walk_dependents(v, propagate_stop,
1757                             (void *)RERR_RESTART);
1758                         break;
1759                 }

1760                 /* FALLTHROUGH */
1761 
1762         case GVT_SVC:
1763                 graph_walk_dependents(v, propagate_start, NULL);
1764                 break;
1765 
1766         case GVT_FILE:
1767 #ifndef NDEBUG
1768                 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1769                     __FILE__, __LINE__);
1770 #endif
1771                 abort();
1772                 /* NOTREACHED */
1773 
1774         default:
1775 #ifndef NDEBUG
1776                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1777                     v->gv_type);
1778 #endif
1779                 abort();
1780         }
1781 }
1782 









1783 static void
1784 propagate_stop(graph_vertex_t *v, void *arg)
1785 {
1786         graph_edge_t *e;
1787         graph_vertex_t *svc;
1788         restarter_error_t err = (restarter_error_t)arg;
1789 



1790         switch (v->gv_type) {
1791         case GVT_INST:
1792                 /* Restarter */
1793                 if (err > RERR_NONE && inst_running(v)) {
1794                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1795                                 vertex_send_event(v,
1796                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1797                         } else {
1798                                 vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1799                         }
1800                 }
1801                 break;
1802 
1803         case GVT_SVC:
1804                 graph_walk_dependents(v, propagate_stop, arg);
1805                 break;
1806 
1807         case GVT_FILE:
1808 #ifndef NDEBUG
1809                 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1810                     __FILE__, __LINE__);
1811 #endif
1812                 abort();
1813                 /* NOTREACHED */
1814 
1815         case GVT_GROUP:
1816                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1817                         graph_walk_dependents(v, propagate_start, NULL);
1818                         break;
1819                 }
1820 
1821                 if (err == RERR_NONE || err > v->gv_restart)
1822                         break;
1823 
1824                 assert(uu_list_numnodes(v->gv_dependents) == 1);
1825                 e = uu_list_first(v->gv_dependents);
1826                 svc = e->ge_vertex;
1827 
1828                 if (inst_running(svc)) {
1829                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1830                                 vertex_send_event(svc,
1831                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1832                         } else {
1833                                 vertex_send_event(svc,
1834                                     RESTARTER_EVENT_TYPE_STOP);
1835                         }
1836                 }
1837                 break;
1838 
1839         default:
1840 #ifndef NDEBUG
1841                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1842                     v->gv_type);
1843 #endif
1844                 abort();
1845         }
1846 }
1847 
1848 void
1849 offline_vertex(graph_vertex_t *v)
1850 {
1851         scf_handle_t *h = libscf_handle_create_bound_loop();
1852         scf_instance_t *scf_inst = safe_scf_instance_create(h);
1853         scf_propertygroup_t *pg = safe_scf_pg_create(h);
1854         restarter_instance_state_t state, next_state;
1855         int r;
1856 


4381         graph_vertex_t *vv;
4382         graph_edge_t *e;
4383 
4384         assert(MUTEX_HELD(&dgraph_lock));
4385 
4386         for (e = uu_list_first(v->gv_dependents); e != NULL;
4387             e = uu_list_next(v->gv_dependents, e)) {
4388                 vv = e->ge_vertex;
4389                 if (vv->gv_type == GVT_INST) {
4390                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4391                                 continue;
4392 
4393                         if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4394                                 continue;
4395 
4396                         if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4397                             (vv->gv_state == RESTARTER_STATE_DEGRADED))
4398                                 return (B_FALSE);
4399                 } else {
4400                         /*
4401                          * Skip all excluded and optional_all dependencies
4402                          * and decide whether to offline the service based
4403                          * on restart_on attribute.
4404                          */
4405                         if (is_depgrp_bypassed(vv))
4406                                 continue;
4407 
4408                         /*
4409                          * For dependency groups or service vertices, keep
4410                          * traversing to see if instances are running.
4411                          */
4412                         if (insubtree_dependents_down(vv) == B_FALSE)
4413                                 return (B_FALSE);
4414                 }
4415         }
4416 
4417         return (B_TRUE);
4418 }
4419 
4420 /*
4421  * Returns true only if none of this service's dependents are 'up' -- online,
4422  * degraded, or offline.
4423  */


4948  * Propagate a start, stop event, or a satisfiability event.
4949  *
4950  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4951  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4952  * full dependent graph to check for newly satisfied nodes.  This is
4953  * necessary for cases when non-direct dependents may be effected but direct
4954  * dependents may not (e.g. for optional_all evaluations, see the
4955  * propagate_satbility() comments).
4956  *
4957  * PROPAGATE_SAT should be used whenever a non-running service moves into
4958  * a state which can satisfy optional dependencies, like disabled or
4959  * maintenance.
4960  */
4961 void
4962 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4963     restarter_error_t rerr)
4964 {
4965         if (type == PROPAGATE_STOP) {
4966                 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4967         } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4968                 graph_walk_dependents(v, propagate_start, NULL);
4969 
4970                 if (type == PROPAGATE_SAT)
4971                         propagate_satbility(v);
4972         } else {
4973 #ifndef NDEBUG
4974                 uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4975                     __LINE__, type);
4976 #endif
4977                 abort();
4978         }
4979 }
4980 
4981 /*
4982  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4983  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4984  * all property group dependencies, and the dependency on the restarter,
4985  * disposing of vertices as appropriate.  If other vertices depend on this
4986  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4987  * returns 0.
4988  */


5019                 err = uu_list_walk(v->gv_dependencies,
5020                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
5021                 assert(err == 0);
5022         }
5023 
5024         delete_instance_dependencies(v, B_TRUE);
5025 
5026         /*
5027          * Deleting an instance can both satisfy and unsatisfy dependencies,
5028          * depending on their type.  First propagate the stop as a RERR_RESTART
5029          * event -- deletion isn't a fault, just a normal stop.  This gives
5030          * dependent services the chance to do a clean shutdown.  Then, mark
5031          * the service as unconfigured and propagate the start event for the
5032          * optional_all dependencies that might have become satisfied.
5033          */
5034         graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
5035 
5036         v->gv_flags &= ~GV_CONFIGURED;
5037         v->gv_flags &= ~GV_DEATHROW;
5038 
5039         graph_walk_dependents(v, propagate_start, NULL);
5040         propagate_satbility(v);
5041 
5042         /*
5043          * If there are no (non-service) dependents, the vertex can be
5044          * completely removed.
5045          */
5046         if (v != milestone && v->gv_refs == 0 &&
5047             uu_list_numnodes(v->gv_dependents) == 1)
5048                 remove_inst_vertex(v);
5049 
5050         if (milestone > MILESTONE_NONE) {
5051                 void *cookie = NULL;
5052 
5053                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5054                         v = e->ge_vertex;
5055 
5056                         if (vertex_unref(v) == VERTEX_INUSE)
5057                                 while (eval_subgraph(v, h) == ECONNABORTED)
5058                                         libscf_handle_rebind(h);
5059 


5444         graph_vertex_t *v;
5445         int r;
5446 
5447         v = e->ge_vertex;
5448 
5449         /* If it's already in the subgraph, skip. */
5450         if (v->gv_flags & GV_TOOFFLINE)
5451                 return (UU_WALK_NEXT);
5452 
5453         switch (v->gv_type) {
5454         case GVT_INST:
5455                 /* If the instance is already disabled, skip it. */
5456                 if (!(v->gv_flags & GV_ENABLED))
5457                         return (UU_WALK_NEXT);
5458 
5459                 v->gv_flags |= GV_TOOFFLINE;
5460                 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5461                 break;
5462         case GVT_GROUP:
5463                 /*
5464                  * Skip all excluded and optional_all dependencies and decide
5465                  * whether to offline the service based on restart_on attribute.
5466                  */
5467                 if (is_depgrp_bypassed(v))
5468                         return (UU_WALK_NEXT);
5469                 break;
5470         }
5471 
5472         r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5473             0);
5474         assert(r == 0);
5475         return (UU_WALK_NEXT);
5476 }
5477 
5478 static int
5479 mark_subgraph(graph_edge_t *e, void *arg)
5480 {
5481         graph_vertex_t *v;
5482         int r;
5483         int optional = (int)arg;
5484 
5485         v = e->ge_vertex;




   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
  25  * Copyright 2016 RackTop Systems.
  26  */
  27 
  28 /*
  29  * graph.c - master restarter graph engine
  30  *
  31  *   The graph engine keeps a dependency graph of all service instances on the
  32  *   system, as recorded in the repository.  It decides when services should
  33  *   be brought up or down based on service states and dependencies and sends
  34  *   commands to restarters to effect any changes.  It also executes
  35  *   administrator commands sent by svcadm via the repository.
  36  *
  37  *   The graph is stored in uu_list_t *dgraph and its vertices are
  38  *   graph_vertex_t's, each of which has a name and an integer id unique to
  39  *   its name (see dict.c).  A vertex's type attribute designates the type
  40  *   of object it represents: GVT_INST for service instances, GVT_SVC for
  41  *   service objects (since service instances may depend on another service,
  42  *   rather than service instance), GVT_FILE for files (which services may
  43  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
  44  *   vertices are necessary because dependency lists may have particular
  45  *   grouping types (require any, require all, optional, or exclude) and


 191  *     access to stn_global
 192  */
 193 int32_t stn_global;
 194 /*
 195  * info_events_all holds a flag to override notification parameters and send
 196  * Information events for all state transitions.
 197  * same about the need of a mutex here.
 198  */
 199 int info_events_all;
 200 
 201 /*
 202  * Services in these states are not considered 'down' by the
 203  * milestone/shutdown code.
 204  */
 205 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
 206         (state) == RESTARTER_STATE_DEGRADED || \
 207         (state) == RESTARTER_STATE_OFFLINE)
 208 
 209 #define is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
 210         ((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \

 211         (v->gv_restart < RERR_RESTART)))
 212 
 213 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
 214 static uu_list_t *dgraph;
 215 static pthread_mutex_t dgraph_lock;
 216 
 217 /*
 218  * milestone indicates the current subgraph.  When NULL, it is the entire
 219  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
 220  * services on which the target vertex depends.
 221  */
 222 static graph_vertex_t *milestone = NULL;
 223 static boolean_t initial_milestone_set = B_FALSE;
 224 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
 225 
 226 /* protected by dgraph_lock */
 227 static boolean_t sulogin_thread_running = B_FALSE;
 228 static boolean_t sulogin_running = B_FALSE;
 229 static boolean_t console_login_ready = B_FALSE;
 230 


1311  */
1312 static int
1313 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1314 {
1315         graph_edge_t *edge;
1316         graph_vertex_t *v;
1317         boolean_t any_qualified;
1318         boolean_t any_unsatisfied;
1319         int i;
1320 
1321         any_qualified = B_FALSE;
1322         any_unsatisfied = B_FALSE;
1323 
1324         for (edge = uu_list_first(groupv->gv_dependencies);
1325             edge != NULL;
1326             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1327                 v = edge->ge_vertex;
1328 
1329                 switch (v->gv_type) {
1330                 case GVT_INST:
1331                         /* Skip missing instances */
1332                         if ((v->gv_flags & GV_CONFIGURED) == 0)

1333                                 continue;
1334 
1335                         if (v->gv_state == RESTARTER_STATE_MAINT)
1336                                 continue;
1337 



1338                         any_qualified = B_TRUE;
1339                         if (v->gv_state == RESTARTER_STATE_OFFLINE) {
1340                                 /*
1341                                  * For offline dependencies, treat unsatisfiable
1342                                  * as satisfied.
1343                                  */
1344                                 i = dependency_satisfied(v, B_TRUE);
1345                                 if (i == -1)
1346                                         i = 1;
1347                         } else if (v->gv_state == RESTARTER_STATE_DISABLED) {
1348                                 /*
1349                                  * If the instance is transitioning out of
1350                                  * disabled the dependency is temporarily
1351                                  * unsatisfied (not unsatisfiable).
1352                                  */
1353                                 i = v->gv_flags & GV_ENABLED ? 0 : 1;
1354                         } else {
1355                                 i = dependency_satisfied(v, satbility);
1356                         }
1357                         break;
1358 
1359                 case GVT_FILE:
1360                         any_qualified = B_TRUE;
1361                         i = dependency_satisfied(v, satbility);
1362 
1363                         break;
1364 
1365                 case GVT_SVC: {






















































1366                         any_qualified = B_TRUE;
1367                         i = optional_all_satisfied(v, satbility);
1368 





1369                         break;
1370                 }
1371 
1372                 case GVT_GROUP:
1373                 default:
1374 #ifndef NDEBUG
1375                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1376                             __LINE__, v->gv_type);
1377 #endif
1378                         abort();
1379                 }
1380 
1381                 if (i == 1)
1382                         continue;
1383 
1384                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1385                     "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1386                     v->gv_name, i == 0 ? "ed" : "able");
1387 
1388                 if (!satbility)


1528  * Decide whether v can satisfy a dependency.  v can either be a child of
1529  * a group vertex, or of an instance vertex.
1530  */
1531 static int
1532 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1533 {
1534         switch (v->gv_type) {
1535         case GVT_INST:
1536                 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1537                         if (v->gv_flags & GV_DEATHROW) {
1538                                 /*
1539                                  * A dependency on an instance with GV_DEATHROW
1540                                  * flag is always considered as satisfied.
1541                                  */
1542                                 return (1);
1543                         }
1544                         return (-1);
1545                 }
1546 
1547                 /*
1548                  * Any vertex with the GV_TODISABLE flag set is guaranteed
1549                  * to have its dependencies unsatisfiable.  Any vertex with
1550                  * GV_TOOFFLINE may be satisfied after it transitions.
1551                  */
1552                 if (v->gv_flags & GV_TODISABLE)
1553                         return (-1);
1554                 if (v->gv_flags & GV_TOOFFLINE)
1555                         return (0);
1556 
1557                 switch (v->gv_state) {
1558                 case RESTARTER_STATE_ONLINE:
1559                 case RESTARTER_STATE_DEGRADED:
1560                         return (1);
1561 
1562                 case RESTARTER_STATE_OFFLINE:
1563                         if (!satbility)
1564                                 return (0);
1565                         return (instance_satisfied(v, satbility) != -1 ?
1566                             0 : -1);
1567 
1568                 case RESTARTER_STATE_DISABLED:
1569                 case RESTARTER_STATE_MAINT:
1570                         return (-1);
1571 
1572                 case RESTARTER_STATE_UNINIT:
1573                         return (0);
1574 
1575                 default:


1651  *      |  A  |--------------->|  B  |-------------->|  C  |
1652  *      +-----+                +-----+               +-----+
1653  *
1654  *                                              offline -> maintenance
1655  *
1656  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1657  * an optional dependency, what was previously an unsatisfiable situation is now
1658  * satisfied (B will never come online, even though its state hasn't changed).
1659  *
1660  * Note that it's not necessary to continue examining dependents after reaching
1661  * an optional_all dependency.  It's not possible for an optional_all dependency
1662  * to change satisfiability without also coming online, in which case we get a
1663  * start event and propagation continues naturally.  However, it does no harm to
1664  * continue propagating satisfiability (as it is a relatively rare event), and
1665  * keeps the walker code simple and generic.
1666  */
1667 /*ARGSUSED*/
1668 static int
1669 satbility_cb(graph_vertex_t *v, void *arg)
1670 {
1671         if (v->gv_flags & GV_TOOFFLINE)
1672                 return (UU_WALK_NEXT);
1673 
1674         if (v->gv_type == GVT_INST)
1675                 graph_start_if_satisfied(v);
1676 
1677         return (UU_WALK_NEXT);
1678 }
1679 
1680 static void
1681 propagate_satbility(graph_vertex_t *v)
1682 {
1683         graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1684 }
1685 
1686 static void propagate_stop(graph_vertex_t *, void *);
1687 
1688 /*
1689  * propagate_start()
1690  *
1691  * This function is used to propagate a start event to the dependents of the
1692  * given vertex.  Any dependents that are offline but have their dependencies
1693  * satisfied are started.  Any dependents that are online and have restart_on
1694  * set to "restart" or "refresh" are restarted because their dependencies have
1695  * just changed.  This only happens with optional_all dependencies.
1696  */
1697 static void
1698 propagate_start(graph_vertex_t *v, void *arg)
1699 {
1700         restarter_error_t err = (restarter_error_t)arg;
1701 
1702         if (v->gv_flags & GV_TOOFFLINE)
1703                 return;
1704 
1705         switch (v->gv_type) {
1706         case GVT_INST:
1707                 /* Restarter */
1708                 if (inst_running(v)) {
1709                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1710                                 vertex_send_event(v,
1711                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1712                         }
1713                 } else {
1714                         graph_start_if_satisfied(v);
1715                 }
1716                 break;
1717 
1718         case GVT_GROUP:
1719                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1720                         graph_walk_dependents(v, propagate_stop,
1721                             (void *)RERR_RESTART);
1722                         break;
1723                 }
1724                 err = v->gv_restart;
1725                 /* FALLTHROUGH */
1726 
1727         case GVT_SVC:
1728                 graph_walk_dependents(v, propagate_start, (void *)err);
1729                 break;
1730 
1731         case GVT_FILE:
1732 #ifndef NDEBUG
1733                 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1734                     __FILE__, __LINE__);
1735 #endif
1736                 abort();
1737                 /* NOTREACHED */
1738 
1739         default:
1740 #ifndef NDEBUG
1741                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1742                     v->gv_type);
1743 #endif
1744                 abort();
1745         }
1746 }
1747 
1748 /*
1749  * propagate_stop()
1750  *
1751  * This function is used to propagate a stop event to the dependents of the
1752  * given vertex.  Any dependents that are online (or in degraded state) with
1753  * the restart_on property set to "restart" or "refresh" will be stopped as
1754  * their dependencies have just changed, propagate_start() will start them
1755  * again once their dependencies have been re-satisfied.
1756  */
1757 static void
1758 propagate_stop(graph_vertex_t *v, void *arg)
1759 {


1760         restarter_error_t err = (restarter_error_t)arg;
1761 
1762         if (v->gv_flags & GV_TOOFFLINE)
1763                 return;
1764 
1765         switch (v->gv_type) {
1766         case GVT_INST:
1767                 /* Restarter */
1768                 if (err > RERR_NONE && inst_running(v)) {
1769                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1770                                 vertex_send_event(v,
1771                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1772                         } else {
1773                                 vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1774                         }
1775                 }
1776                 break;
1777 
1778         case GVT_SVC:
1779                 graph_walk_dependents(v, propagate_stop, arg);
1780                 break;
1781 
1782         case GVT_FILE:
1783 #ifndef NDEBUG
1784                 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1785                     __FILE__, __LINE__);
1786 #endif
1787                 abort();
1788                 /* NOTREACHED */
1789 
1790         case GVT_GROUP:
1791                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1792                         graph_walk_dependents(v, propagate_start, NULL);
1793                         break;
1794                 }
1795 
1796                 if (err == RERR_NONE || err > v->gv_restart)
1797                         break;
1798 
1799                 graph_walk_dependents(v, propagate_stop, arg);












1800                 break;
1801 
1802         default:
1803 #ifndef NDEBUG
1804                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1805                     v->gv_type);
1806 #endif
1807                 abort();
1808         }
1809 }
1810 
1811 void
1812 offline_vertex(graph_vertex_t *v)
1813 {
1814         scf_handle_t *h = libscf_handle_create_bound_loop();
1815         scf_instance_t *scf_inst = safe_scf_instance_create(h);
1816         scf_propertygroup_t *pg = safe_scf_pg_create(h);
1817         restarter_instance_state_t state, next_state;
1818         int r;
1819 


4344         graph_vertex_t *vv;
4345         graph_edge_t *e;
4346 
4347         assert(MUTEX_HELD(&dgraph_lock));
4348 
4349         for (e = uu_list_first(v->gv_dependents); e != NULL;
4350             e = uu_list_next(v->gv_dependents, e)) {
4351                 vv = e->ge_vertex;
4352                 if (vv->gv_type == GVT_INST) {
4353                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4354                                 continue;
4355 
4356                         if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4357                                 continue;
4358 
4359                         if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4360                             (vv->gv_state == RESTARTER_STATE_DEGRADED))
4361                                 return (B_FALSE);
4362                 } else {
4363                         /*
4364                          * Skip all excluded dependents and decide whether
4365                          * to offline the service based on the restart_on
4366                          * on attribute.
4367                          */
4368                         if (is_depgrp_bypassed(vv))
4369                                 continue;
4370 
4371                         /*
4372                          * For dependency groups or service vertices, keep
4373                          * traversing to see if instances are running.
4374                          */
4375                         if (insubtree_dependents_down(vv) == B_FALSE)
4376                                 return (B_FALSE);
4377                 }
4378         }
4379 
4380         return (B_TRUE);
4381 }
4382 
4383 /*
4384  * Returns true only if none of this service's dependents are 'up' -- online,
4385  * degraded, or offline.
4386  */


4911  * Propagate a start, stop event, or a satisfiability event.
4912  *
4913  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4914  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4915  * full dependent graph to check for newly satisfied nodes.  This is
4916  * necessary for cases when non-direct dependents may be effected but direct
4917  * dependents may not (e.g. for optional_all evaluations, see the
4918  * propagate_satbility() comments).
4919  *
4920  * PROPAGATE_SAT should be used whenever a non-running service moves into
4921  * a state which can satisfy optional dependencies, like disabled or
4922  * maintenance.
4923  */
4924 void
4925 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4926     restarter_error_t rerr)
4927 {
4928         if (type == PROPAGATE_STOP) {
4929                 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4930         } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4931                 graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
4932 
4933                 if (type == PROPAGATE_SAT)
4934                         propagate_satbility(v);
4935         } else {
4936 #ifndef NDEBUG
4937                 uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4938                     __LINE__, type);
4939 #endif
4940                 abort();
4941         }
4942 }
4943 
4944 /*
4945  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4946  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4947  * all property group dependencies, and the dependency on the restarter,
4948  * disposing of vertices as appropriate.  If other vertices depend on this
4949  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4950  * returns 0.
4951  */


4982                 err = uu_list_walk(v->gv_dependencies,
4983                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
4984                 assert(err == 0);
4985         }
4986 
4987         delete_instance_dependencies(v, B_TRUE);
4988 
4989         /*
4990          * Deleting an instance can both satisfy and unsatisfy dependencies,
4991          * depending on their type.  First propagate the stop as a RERR_RESTART
4992          * event -- deletion isn't a fault, just a normal stop.  This gives
4993          * dependent services the chance to do a clean shutdown.  Then, mark
4994          * the service as unconfigured and propagate the start event for the
4995          * optional_all dependencies that might have become satisfied.
4996          */
4997         graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
4998 
4999         v->gv_flags &= ~GV_CONFIGURED;
5000         v->gv_flags &= ~GV_DEATHROW;
5001 
5002         graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
5003         propagate_satbility(v);
5004 
5005         /*
5006          * If there are no (non-service) dependents, the vertex can be
5007          * completely removed.
5008          */
5009         if (v != milestone && v->gv_refs == 0 &&
5010             uu_list_numnodes(v->gv_dependents) == 1)
5011                 remove_inst_vertex(v);
5012 
5013         if (milestone > MILESTONE_NONE) {
5014                 void *cookie = NULL;
5015 
5016                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5017                         v = e->ge_vertex;
5018 
5019                         if (vertex_unref(v) == VERTEX_INUSE)
5020                                 while (eval_subgraph(v, h) == ECONNABORTED)
5021                                         libscf_handle_rebind(h);
5022 


5407         graph_vertex_t *v;
5408         int r;
5409 
5410         v = e->ge_vertex;
5411 
5412         /* If it's already in the subgraph, skip. */
5413         if (v->gv_flags & GV_TOOFFLINE)
5414                 return (UU_WALK_NEXT);
5415 
5416         switch (v->gv_type) {
5417         case GVT_INST:
5418                 /* If the instance is already disabled, skip it. */
5419                 if (!(v->gv_flags & GV_ENABLED))
5420                         return (UU_WALK_NEXT);
5421 
5422                 v->gv_flags |= GV_TOOFFLINE;
5423                 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5424                 break;
5425         case GVT_GROUP:
5426                 /*
5427                  * Skip all excluded dependents and decide whether to offline
5428                  * the service based on the restart_on attribute.
5429                  */
5430                 if (is_depgrp_bypassed(v))
5431                         return (UU_WALK_NEXT);
5432                 break;
5433         }
5434 
5435         r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5436             0);
5437         assert(r == 0);
5438         return (UU_WALK_NEXT);
5439 }
5440 
5441 static int
5442 mark_subgraph(graph_edge_t *e, void *arg)
5443 {
5444         graph_vertex_t *v;
5445         int r;
5446         int optional = (int)arg;
5447 
5448         v = e->ge_vertex;