Print this page
7267 SMF is fast and loose with optional dependencies
Reviewed by: Dan McDonald <danmcd@omniti.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Albert Lee <trisk@omniti.com>
Reviewed by: Gordon Ross <gordon.w.ross@gmail.com>


   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
  25  * Copyright 2016 Toomas Soome <tsoome@me.com>

  26  */
  27 
  28 /*
  29  * graph.c - master restarter graph engine
  30  *
  31  *   The graph engine keeps a dependency graph of all service instances on the
  32  *   system, as recorded in the repository.  It decides when services should
  33  *   be brought up or down based on service states and dependencies and sends
  34  *   commands to restarters to effect any changes.  It also executes
  35  *   administrator commands sent by svcadm via the repository.
  36  *
  37  *   The graph is stored in uu_list_t *dgraph and its vertices are
  38  *   graph_vertex_t's, each of which has a name and an integer id unique to
  39  *   its name (see dict.c).  A vertex's type attribute designates the type
  40  *   of object it represents: GVT_INST for service instances, GVT_SVC for
  41  *   service objects (since service instances may depend on another service,
  42  *   rather than service instance), GVT_FILE for files (which services may
  43  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
  44  *   vertices are necessary because dependency lists may have particular
  45  *   grouping types (require any, require all, optional, or exclude) and


 191  *     access to stn_global
 192  */
 193 int32_t stn_global;
 194 /*
 195  * info_events_all holds a flag to override notification parameters and send
 196  * Information events for all state transitions.
 197  * same about the need of a mutex here.
 198  */
 199 int info_events_all;
 200 
 201 /*
 202  * Services in these states are not considered 'down' by the
 203  * milestone/shutdown code.
 204  */
 205 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
 206         (state) == RESTARTER_STATE_DEGRADED || \
 207         (state) == RESTARTER_STATE_OFFLINE)
 208 
 209 #define is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
 210         ((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \
 211         (v->gv_depgroup == DEPGRP_OPTIONAL_ALL) || \
 212         (v->gv_restart < RERR_RESTART)))
 213 




 214 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
 215 static uu_list_t *dgraph;
 216 static pthread_mutex_t dgraph_lock;
 217 
 218 /*
 219  * milestone indicates the current subgraph.  When NULL, it is the entire
 220  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
 221  * services on which the target vertex depends.
 222  */
 223 static graph_vertex_t *milestone = NULL;
 224 static boolean_t initial_milestone_set = B_FALSE;
 225 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
 226 
 227 /* protected by dgraph_lock */
 228 static boolean_t sulogin_thread_running = B_FALSE;
 229 static boolean_t sulogin_running = B_FALSE;
 230 static boolean_t console_login_ready = B_FALSE;
 231 
 232 /* Number of services to come down to complete milestone transition. */
 233 static uint_t non_subgraph_svcs;


1281 
1282         satisfiable = B_FALSE;
1283 
1284         for (edge = uu_list_first(groupv->gv_dependencies);
1285             edge != NULL;
1286             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1287                 s = dependency_satisfied(edge->ge_vertex, satbility);
1288 
1289                 if (s == 1)
1290                         return (1);
1291 
1292                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1293                     "require_any(%s): %s is unsatisfi%s.\n",
1294                     groupv->gv_name, edge->ge_vertex->gv_name,
1295                     s == 0 ? "ed" : "able");
1296 
1297                 if (satbility && s == 0)
1298                         satisfiable = B_TRUE;
1299         }
1300 
1301         return (!satbility || satisfiable ? 0 : -1);
1302 }
1303 
1304 /*
1305  * An optional_all dependency only considers elements which are configured,
1306  * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1307  * is unsatisfied.
1308  *
1309  * Offline dependencies which are waiting for a dependency to come online are
1310  * unsatisfied.  Offline dependences which cannot possibly come online
1311  * (unsatisfiable) are always considered satisfied.
1312  */
1313 static int
1314 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1315 {
1316         graph_edge_t *edge;
1317         graph_vertex_t *v;
1318         boolean_t any_qualified;
1319         boolean_t any_unsatisfied;
1320         int i;
1321 
1322         any_qualified = B_FALSE;
1323         any_unsatisfied = B_FALSE;
1324 
1325         for (edge = uu_list_first(groupv->gv_dependencies);
1326             edge != NULL;
1327             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1328                 v = edge->ge_vertex;
1329 
1330                 switch (v->gv_type) {
1331                 case GVT_INST:
1332                         /* Skip missing or disabled instances */
1333                         if ((v->gv_flags & (GV_CONFIGURED | GV_ENABLED)) !=
1334                             (GV_CONFIGURED | GV_ENABLED))
1335                                 continue;
1336 
1337                         if (v->gv_state == RESTARTER_STATE_MAINT)
1338                                 continue;
1339 
1340                         if (v->gv_flags & GV_TOOFFLINE)
1341                                 continue;
1342 
1343                         any_qualified = B_TRUE;
1344                         if (v->gv_state == RESTARTER_STATE_OFFLINE) {

1345                                 /*
1346                                  * For offline dependencies, treat unsatisfiable
1347                                  * as satisfied.
1348                                  */
1349                                 i = dependency_satisfied(v, B_TRUE);
1350                                 if (i == -1)
1351                                         i = 1;
1352                         } else if (v->gv_state == RESTARTER_STATE_DISABLED) {
1353                                 /*
1354                                  * The service is enabled, but hasn't
1355                                  * transitioned out of disabled yet.  Treat it
1356                                  * as unsatisfied (not unsatisfiable).
1357                                  */
1358                                 i = 0;
1359                         } else {
1360                                 i = dependency_satisfied(v, satbility);
1361                         }
1362                         break;
1363 
1364                 case GVT_FILE:
1365                         any_qualified = B_TRUE;
1366                         i = dependency_satisfied(v, satbility);
1367 
1368                         break;
1369 
1370                 case GVT_SVC: {
1371                         boolean_t svc_any_qualified;
1372                         boolean_t svc_satisfied;
1373                         boolean_t svc_satisfiable;
1374                         graph_vertex_t *v2;
1375                         graph_edge_t *e2;
1376 
1377                         svc_any_qualified = B_FALSE;
1378                         svc_satisfied = B_FALSE;
1379                         svc_satisfiable = B_FALSE;
1380 
1381                         for (e2 = uu_list_first(v->gv_dependencies);
1382                             e2 != NULL;
1383                             e2 = uu_list_next(v->gv_dependencies, e2)) {
1384                                 v2 = e2->ge_vertex;
1385                                 assert(v2->gv_type == GVT_INST);
1386 
1387                                 if ((v2->gv_flags &
1388                                     (GV_CONFIGURED | GV_ENABLED)) !=
1389                                     (GV_CONFIGURED | GV_ENABLED))
1390                                         continue;
1391 
1392                                 if (v2->gv_state == RESTARTER_STATE_MAINT)
1393                                         continue;
1394 
1395                                 if (v2->gv_flags & GV_TOOFFLINE)
1396                                         continue;
1397 
1398                                 svc_any_qualified = B_TRUE;
1399 
1400                                 if (v2->gv_state == RESTARTER_STATE_OFFLINE) {
1401                                         /*
1402                                          * For offline dependencies, treat
1403                                          * unsatisfiable as satisfied.
1404                                          */
1405                                         i = dependency_satisfied(v2, B_TRUE);
1406                                         if (i == -1)
1407                                                 i = 1;
1408                                 } else if (v2->gv_state ==
1409                                     RESTARTER_STATE_DISABLED) {
1410                                         i = 0;
1411                                 } else {
1412                                         i = dependency_satisfied(v2, satbility);
1413                                 }
1414 
1415                                 if (i == 1) {
1416                                         svc_satisfied = B_TRUE;
1417                                         break;
1418                                 }
1419                                 if (i == 0)
1420                                         svc_satisfiable = B_TRUE;
1421                         }
1422 
1423                         if (!svc_any_qualified)
1424                                 continue;
1425                         any_qualified = B_TRUE;
1426                         if (svc_satisfied) {
1427                                 i = 1;
1428                         } else if (svc_satisfiable) {
1429                                 i = 0;
1430                         } else {
1431                                 i = -1;
1432                         }
1433                         break;
1434                 }
1435 
1436                 case GVT_GROUP:
1437                 default:
1438 #ifndef NDEBUG
1439                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1440                             __LINE__, v->gv_type);
1441 #endif
1442                         abort();
1443                 }
1444 
1445                 if (i == 1)
1446                         continue;
1447 
1448                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1449                     "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1450                     v->gv_name, i == 0 ? "ed" : "able");
1451 
1452                 if (!satbility)
1453                         return (0);
1454                 if (i == -1)
1455                         return (-1);


1592  * Decide whether v can satisfy a dependency.  v can either be a child of
1593  * a group vertex, or of an instance vertex.
1594  */
1595 static int
1596 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1597 {
1598         switch (v->gv_type) {
1599         case GVT_INST:
1600                 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1601                         if (v->gv_flags & GV_DEATHROW) {
1602                                 /*
1603                                  * A dependency on an instance with GV_DEATHROW
1604                                  * flag is always considered as satisfied.
1605                                  */
1606                                 return (1);
1607                         }
1608                         return (-1);
1609                 }
1610 
1611                 /*
1612                  * Any vertex with the GV_TOOFFLINE flag set is guaranteed
1613                  * to have its dependencies unsatisfiable.





1614                  */
1615                 if (v->gv_flags & GV_TOOFFLINE)
1616                         return (-1);
1617 
1618                 switch (v->gv_state) {
1619                 case RESTARTER_STATE_ONLINE:
1620                 case RESTARTER_STATE_DEGRADED:




1621                         return (1);
1622 
1623                 case RESTARTER_STATE_OFFLINE:
1624                         if (!satbility)
1625                                 return (0);
1626                         return (instance_satisfied(v, satbility) != -1 ?
1627                             0 : -1);
1628 
1629                 case RESTARTER_STATE_DISABLED:





1630                 case RESTARTER_STATE_MAINT:
1631                         return (-1);
1632 
1633                 case RESTARTER_STATE_UNINIT:
1634                         return (0);
1635 
1636                 default:
1637 #ifndef NDEBUG
1638                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
1639                             __FILE__, __LINE__, v->gv_state);
1640 #endif
1641                         abort();
1642                         /* NOTREACHED */
1643                 }
1644 
1645         case GVT_SVC:
1646                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1647                         return (-1);
1648                 return (require_any_satisfied(v, satbility));
1649 


1712  *      |  A  |--------------->|  B  |-------------->|  C  |
1713  *      +-----+                +-----+               +-----+
1714  *
1715  *                                              offline -> maintenance
1716  *
1717  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1718  * an optional dependency, what was previously an unsatisfiable situation is now
1719  * satisfied (B will never come online, even though its state hasn't changed).
1720  *
1721  * Note that it's not necessary to continue examining dependents after reaching
1722  * an optional_all dependency.  It's not possible for an optional_all dependency
1723  * to change satisfiability without also coming online, in which case we get a
1724  * start event and propagation continues naturally.  However, it does no harm to
1725  * continue propagating satisfiability (as it is a relatively rare event), and
1726  * keeps the walker code simple and generic.
1727  */
1728 /*ARGSUSED*/
1729 static int
1730 satbility_cb(graph_vertex_t *v, void *arg)
1731 {



1732         if (v->gv_type == GVT_INST)
1733                 graph_start_if_satisfied(v);
1734 
1735         return (UU_WALK_NEXT);
1736 }
1737 
1738 static void
1739 propagate_satbility(graph_vertex_t *v)
1740 {
1741         graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1742 }
1743 
1744 static void propagate_stop(graph_vertex_t *, void *);
1745 
1746 /* ARGSUSED */








1747 static void
1748 propagate_start(graph_vertex_t *v, void *arg)
1749 {





1750         switch (v->gv_type) {
1751         case GVT_INST:







1752                 graph_start_if_satisfied(v);

1753                 break;
1754 
1755         case GVT_GROUP:
1756                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1757                         graph_walk_dependents(v, propagate_stop,
1758                             (void *)RERR_RESTART);
1759                         break;
1760                 }

1761                 /* FALLTHROUGH */
1762 
1763         case GVT_SVC:
1764                 graph_walk_dependents(v, propagate_start, NULL);
1765                 break;
1766 
1767         case GVT_FILE:
1768 #ifndef NDEBUG
1769                 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1770                     __FILE__, __LINE__);
1771 #endif
1772                 abort();
1773                 /* NOTREACHED */
1774 
1775         default:
1776 #ifndef NDEBUG
1777                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1778                     v->gv_type);
1779 #endif
1780                 abort();
1781         }
1782 }
1783 









1784 static void
1785 propagate_stop(graph_vertex_t *v, void *arg)
1786 {
1787         graph_edge_t *e;
1788         graph_vertex_t *svc;
1789         restarter_error_t err = (restarter_error_t)arg;
1790 



1791         switch (v->gv_type) {
1792         case GVT_INST:
1793                 /* Restarter */
1794                 if (err > RERR_NONE && inst_running(v)) {
1795                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1796                                 vertex_send_event(v,
1797                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1798                         } else {
1799                                 vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1800                         }
1801                 }
1802                 break;
1803 
1804         case GVT_SVC:
1805                 graph_walk_dependents(v, propagate_stop, arg);
1806                 break;
1807 
1808         case GVT_FILE:
1809 #ifndef NDEBUG
1810                 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1811                     __FILE__, __LINE__);
1812 #endif
1813                 abort();
1814                 /* NOTREACHED */
1815 
1816         case GVT_GROUP:
1817                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1818                         graph_walk_dependents(v, propagate_start, NULL);

1819                         break;
1820                 }
1821 
1822                 if (err == RERR_NONE || err > v->gv_restart)
1823                         break;
1824 
1825                 assert(uu_list_numnodes(v->gv_dependents) == 1);
1826                 e = uu_list_first(v->gv_dependents);
1827                 svc = e->ge_vertex;
1828 
1829                 if (inst_running(svc)) {
1830                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1831                                 vertex_send_event(svc,
1832                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1833                         } else {
1834                                 vertex_send_event(svc,
1835                                     RESTARTER_EVENT_TYPE_STOP);
1836                         }
1837                 }
1838                 break;
1839 
1840         default:
1841 #ifndef NDEBUG
1842                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1843                     v->gv_type);
1844 #endif
1845                 abort();
1846         }
1847 }
1848 
1849 void
1850 offline_vertex(graph_vertex_t *v)
1851 {
1852         scf_handle_t *h = libscf_handle_create_bound_loop();
1853         scf_instance_t *scf_inst = safe_scf_instance_create(h);
1854         scf_propertygroup_t *pg = safe_scf_pg_create(h);
1855         restarter_instance_state_t state, next_state;
1856         int r;
1857 


4376         graph_vertex_t *vv;
4377         graph_edge_t *e;
4378 
4379         assert(MUTEX_HELD(&dgraph_lock));
4380 
4381         for (e = uu_list_first(v->gv_dependents); e != NULL;
4382             e = uu_list_next(v->gv_dependents, e)) {
4383                 vv = e->ge_vertex;
4384                 if (vv->gv_type == GVT_INST) {
4385                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4386                                 continue;
4387 
4388                         if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4389                                 continue;
4390 
4391                         if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4392                             (vv->gv_state == RESTARTER_STATE_DEGRADED))
4393                                 return (B_FALSE);
4394                 } else {
4395                         /*
4396                          * Skip all excluded and optional_all dependencies
4397                          * and decide whether to offline the service based
4398                          * on restart_on attribute.
4399                          */
4400                         if (is_depgrp_bypassed(vv))
4401                                 continue;
4402 
4403                         /*
4404                          * For dependency groups or service vertices, keep
4405                          * traversing to see if instances are running.
4406                          */
4407                         if (insubtree_dependents_down(vv) == B_FALSE)
4408                                 return (B_FALSE);
4409                 }
4410         }
4411 
4412         return (B_TRUE);
4413 }
4414 
4415 /*
4416  * Returns true only if none of this service's dependents are 'up' -- online,
4417  * degraded, or offline.
4418  */


4943  * Propagate a start, stop event, or a satisfiability event.
4944  *
4945  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4946  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4947  * full dependent graph to check for newly satisfied nodes.  This is
4948  * necessary for cases when non-direct dependents may be effected but direct
4949  * dependents may not (e.g. for optional_all evaluations, see the
4950  * propagate_satbility() comments).
4951  *
4952  * PROPAGATE_SAT should be used whenever a non-running service moves into
4953  * a state which can satisfy optional dependencies, like disabled or
4954  * maintenance.
4955  */
4956 void
4957 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4958     restarter_error_t rerr)
4959 {
4960         if (type == PROPAGATE_STOP) {
4961                 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4962         } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4963                 graph_walk_dependents(v, propagate_start, NULL);
4964 
4965                 if (type == PROPAGATE_SAT)
4966                         propagate_satbility(v);
4967         } else {
4968 #ifndef NDEBUG
4969                 uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4970                     __LINE__, type);
4971 #endif
4972                 abort();
4973         }
4974 }
4975 
4976 /*
4977  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4978  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4979  * all property group dependencies, and the dependency on the restarter,
4980  * disposing of vertices as appropriate.  If other vertices depend on this
4981  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4982  * returns 0.
4983  */


5014                 err = uu_list_walk(v->gv_dependencies,
5015                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
5016                 assert(err == 0);
5017         }
5018 
5019         delete_instance_dependencies(v, B_TRUE);
5020 
5021         /*
5022          * Deleting an instance can both satisfy and unsatisfy dependencies,
5023          * depending on their type.  First propagate the stop as a RERR_RESTART
5024          * event -- deletion isn't a fault, just a normal stop.  This gives
5025          * dependent services the chance to do a clean shutdown.  Then, mark
5026          * the service as unconfigured and propagate the start event for the
5027          * optional_all dependencies that might have become satisfied.
5028          */
5029         graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
5030 
5031         v->gv_flags &= ~GV_CONFIGURED;
5032         v->gv_flags &= ~GV_DEATHROW;
5033 
5034         graph_walk_dependents(v, propagate_start, NULL);
5035         propagate_satbility(v);
5036 
5037         /*
5038          * If there are no (non-service) dependents, the vertex can be
5039          * completely removed.
5040          */
5041         if (v != milestone && v->gv_refs == 0 &&
5042             uu_list_numnodes(v->gv_dependents) == 1)
5043                 remove_inst_vertex(v);
5044 
5045         if (milestone > MILESTONE_NONE) {
5046                 void *cookie = NULL;
5047 
5048                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5049                         v = e->ge_vertex;
5050 
5051                         if (vertex_unref(v) == VERTEX_INUSE)
5052                                 while (eval_subgraph(v, h) == ECONNABORTED)
5053                                         libscf_handle_rebind(h);
5054 


5430 
5431 /*
5432  * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5433  * to the instances that are supposed to go offline during an
5434  * administrative disable operation.
5435  */
5436 static int
5437 mark_subtree(graph_edge_t *e, void *arg)
5438 {
5439         graph_vertex_t *v;
5440         int r;
5441 
5442         v = e->ge_vertex;
5443 
5444         /* If it's already in the subgraph, skip. */
5445         if (v->gv_flags & GV_TOOFFLINE)
5446                 return (UU_WALK_NEXT);
5447 
5448         switch (v->gv_type) {
5449         case GVT_INST:
5450                 /* If the instance is already disabled, skip it. */
5451                 if (!(v->gv_flags & GV_ENABLED))
5452                         return (UU_WALK_NEXT);
5453 
5454                 v->gv_flags |= GV_TOOFFLINE;
5455                 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5456                 break;
5457         case GVT_GROUP:
5458                 /*
5459                  * Skip all excluded and optional_all dependencies and decide
5460                  * whether to offline the service based on restart_on attribute.
5461                  */
5462                 if (is_depgrp_bypassed(v))
5463                         return (UU_WALK_NEXT);
5464                 break;
5465         }
5466 
5467         r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5468             0);
5469         assert(r == 0);
5470         return (UU_WALK_NEXT);
5471 }
5472 
5473 static int
5474 mark_subgraph(graph_edge_t *e, void *arg)
5475 {
5476         graph_vertex_t *v;
5477         int r;
5478         int optional = (int)arg;
5479 
5480         v = e->ge_vertex;




   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
  25  * Copyright 2016 Toomas Soome <tsoome@me.com>
  26  * Copyright 2016 RackTop Systems.
  27  */
  28 
  29 /*
  30  * graph.c - master restarter graph engine
  31  *
  32  *   The graph engine keeps a dependency graph of all service instances on the
  33  *   system, as recorded in the repository.  It decides when services should
  34  *   be brought up or down based on service states and dependencies and sends
  35  *   commands to restarters to effect any changes.  It also executes
  36  *   administrator commands sent by svcadm via the repository.
  37  *
  38  *   The graph is stored in uu_list_t *dgraph and its vertices are
  39  *   graph_vertex_t's, each of which has a name and an integer id unique to
  40  *   its name (see dict.c).  A vertex's type attribute designates the type
  41  *   of object it represents: GVT_INST for service instances, GVT_SVC for
  42  *   service objects (since service instances may depend on another service,
  43  *   rather than service instance), GVT_FILE for files (which services may
  44  *   depend on), and GVT_GROUP for dependencies on multiple objects.  GVT_GROUP
  45  *   vertices are necessary because dependency lists may have particular
  46  *   grouping types (require any, require all, optional, or exclude) and


 192  *     access to stn_global
 193  */
 194 int32_t stn_global;
 195 /*
 196  * info_events_all holds a flag to override notification parameters and send
 197  * Information events for all state transitions.
 198  * same about the need of a mutex here.
 199  */
 200 int info_events_all;
 201 
 202 /*
 203  * Services in these states are not considered 'down' by the
 204  * milestone/shutdown code.
 205  */
 206 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
 207         (state) == RESTARTER_STATE_DEGRADED || \
 208         (state) == RESTARTER_STATE_OFFLINE)
 209 
 210 #define is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
 211         ((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \

 212         (v->gv_restart < RERR_RESTART)))
 213 
 214 #define is_inst_bypassed(v) ((v->gv_type == GVT_INST) && \
 215         ((v->gv_flags & GV_TODISABLE) || \
 216         (v->gv_flags & GV_TOOFFLINE)))
 217 
 218 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
 219 static uu_list_t *dgraph;
 220 static pthread_mutex_t dgraph_lock;
 221 
 222 /*
 223  * milestone indicates the current subgraph.  When NULL, it is the entire
 224  * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
 225  * services on which the target vertex depends.
 226  */
 227 static graph_vertex_t *milestone = NULL;
 228 static boolean_t initial_milestone_set = B_FALSE;
 229 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
 230 
 231 /* protected by dgraph_lock */
 232 static boolean_t sulogin_thread_running = B_FALSE;
 233 static boolean_t sulogin_running = B_FALSE;
 234 static boolean_t console_login_ready = B_FALSE;
 235 
 236 /* Number of services to come down to complete milestone transition. */
 237 static uint_t non_subgraph_svcs;


1285 
1286         satisfiable = B_FALSE;
1287 
1288         for (edge = uu_list_first(groupv->gv_dependencies);
1289             edge != NULL;
1290             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1291                 s = dependency_satisfied(edge->ge_vertex, satbility);
1292 
1293                 if (s == 1)
1294                         return (1);
1295 
1296                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1297                     "require_any(%s): %s is unsatisfi%s.\n",
1298                     groupv->gv_name, edge->ge_vertex->gv_name,
1299                     s == 0 ? "ed" : "able");
1300 
1301                 if (satbility && s == 0)
1302                         satisfiable = B_TRUE;
1303         }
1304 
1305         return ((!satbility || satisfiable) ? 0 : -1);
1306 }
1307 
1308 /*
1309  * An optional_all dependency only considers elements which are configured,
1310  * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1311  * is unsatisfied.
1312  *
1313  * Offline dependencies which are waiting for a dependency to come online are
1314  * unsatisfied.  Offline dependences which cannot possibly come online
1315  * (unsatisfiable) are always considered satisfied.
1316  */
1317 static int
1318 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1319 {
1320         graph_edge_t *edge;
1321         graph_vertex_t *v;
1322         boolean_t any_qualified;
1323         boolean_t any_unsatisfied;
1324         int i;
1325 
1326         any_qualified = B_FALSE;
1327         any_unsatisfied = B_FALSE;
1328 
1329         for (edge = uu_list_first(groupv->gv_dependencies);
1330             edge != NULL;
1331             edge = uu_list_next(groupv->gv_dependencies, edge)) {
1332                 v = edge->ge_vertex;
1333 
1334                 switch (v->gv_type) {
1335                 case GVT_INST:
1336                         /* Skip missing instances */
1337                         if ((v->gv_flags & GV_CONFIGURED) == 0)

1338                                 continue;
1339 
1340                         if (v->gv_state == RESTARTER_STATE_MAINT)
1341                                 continue;
1342 



1343                         any_qualified = B_TRUE;
1344                         if (v->gv_state == RESTARTER_STATE_OFFLINE ||
1345                             v->gv_state == RESTARTER_STATE_DISABLED) {
1346                                 /*
1347                                  * For offline/disabled dependencies,
1348                                  * treat unsatisfiable as satisfied.
1349                                  */
1350                                 i = dependency_satisfied(v, B_TRUE);
1351                                 if (i == -1)
1352                                         i = 1;







1353                         } else {
1354                                 i = dependency_satisfied(v, satbility);
1355                         }
1356                         break;
1357 
1358                 case GVT_FILE:
1359                         any_qualified = B_TRUE;
1360                         i = dependency_satisfied(v, satbility);
1361 
1362                         break;
1363 
1364                 case GVT_SVC: {
1365                         any_qualified = B_TRUE;
1366                         i = optional_all_satisfied(v, satbility);



1367 








































1368                         break;
1369                 }



1370 













1371                 case GVT_GROUP:
1372                 default:
1373 #ifndef NDEBUG
1374                         uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1375                             __LINE__, v->gv_type);
1376 #endif
1377                         abort();
1378                 }
1379 
1380                 if (i == 1)
1381                         continue;
1382 
1383                 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1384                     "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1385                     v->gv_name, i == 0 ? "ed" : "able");
1386 
1387                 if (!satbility)
1388                         return (0);
1389                 if (i == -1)
1390                         return (-1);


1527  * Decide whether v can satisfy a dependency.  v can either be a child of
1528  * a group vertex, or of an instance vertex.
1529  */
1530 static int
1531 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1532 {
1533         switch (v->gv_type) {
1534         case GVT_INST:
1535                 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1536                         if (v->gv_flags & GV_DEATHROW) {
1537                                 /*
1538                                  * A dependency on an instance with GV_DEATHROW
1539                                  * flag is always considered as satisfied.
1540                                  */
1541                                 return (1);
1542                         }
1543                         return (-1);
1544                 }
1545 
1546                 /*
1547                  * Vertices may be transitioning so we try to figure out if
1548                  * the end state is likely to satisfy the dependency instead
1549                  * of assuming the dependency is unsatisfied/unsatisfiable.
1550                  *
1551                  * Support for optional_all dependencies depends on us getting
1552                  * this right because unsatisfiable dependencies are treated
1553                  * as being satisfied.
1554                  */



1555                 switch (v->gv_state) {
1556                 case RESTARTER_STATE_ONLINE:
1557                 case RESTARTER_STATE_DEGRADED:
1558                         if (v->gv_flags & GV_TODISABLE)
1559                                 return (-1);
1560                         if (v->gv_flags & GV_TOOFFLINE)
1561                                 return (0);
1562                         return (1);
1563 
1564                 case RESTARTER_STATE_OFFLINE:
1565                         if (!satbility || v->gv_flags & GV_TODISABLE)
1566                                 return (satbility ? -1 : 0);
1567                         return (instance_satisfied(v, satbility) != -1 ?
1568                             0 : -1);
1569 
1570                 case RESTARTER_STATE_DISABLED:
1571                         if (!satbility || !(v->gv_flags & GV_ENABLED))
1572                                 return (satbility ? -1 : 0);
1573                         return (instance_satisfied(v, satbility) != -1 ?
1574                             0 : -1);
1575 
1576                 case RESTARTER_STATE_MAINT:
1577                         return (-1);
1578 
1579                 case RESTARTER_STATE_UNINIT:
1580                         return (0);
1581 
1582                 default:
1583 #ifndef NDEBUG
1584                         uu_warn("%s:%d: Unexpected vertex state %d.\n",
1585                             __FILE__, __LINE__, v->gv_state);
1586 #endif
1587                         abort();
1588                         /* NOTREACHED */
1589                 }
1590 
1591         case GVT_SVC:
1592                 if (uu_list_numnodes(v->gv_dependencies) == 0)
1593                         return (-1);
1594                 return (require_any_satisfied(v, satbility));
1595 


1658  *      |  A  |--------------->|  B  |-------------->|  C  |
1659  *      +-----+                +-----+               +-----+
1660  *
1661  *                                              offline -> maintenance
1662  *
1663  * If C goes into maintenance, it's not enough simply to check B.  Because A has
1664  * an optional dependency, what was previously an unsatisfiable situation is now
1665  * satisfied (B will never come online, even though its state hasn't changed).
1666  *
1667  * Note that it's not necessary to continue examining dependents after reaching
1668  * an optional_all dependency.  It's not possible for an optional_all dependency
1669  * to change satisfiability without also coming online, in which case we get a
1670  * start event and propagation continues naturally.  However, it does no harm to
1671  * continue propagating satisfiability (as it is a relatively rare event), and
1672  * keeps the walker code simple and generic.
1673  */
1674 /*ARGSUSED*/
1675 static int
1676 satbility_cb(graph_vertex_t *v, void *arg)
1677 {
1678         if (is_inst_bypassed(v))
1679                 return (UU_WALK_NEXT);
1680 
1681         if (v->gv_type == GVT_INST)
1682                 graph_start_if_satisfied(v);
1683 
1684         return (UU_WALK_NEXT);
1685 }
1686 
1687 static void
1688 propagate_satbility(graph_vertex_t *v)
1689 {
1690         graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1691 }
1692 
1693 static void propagate_stop(graph_vertex_t *, void *);
1694 
1695 /*
1696  * propagate_start()
1697  *
1698  * This function is used to propagate a start event to the dependents of the
1699  * given vertex.  Any dependents that are offline but have their dependencies
1700  * satisfied are started.  Any dependents that are online and have restart_on
1701  * set to "restart" or "refresh" are restarted because their dependencies have
1702  * just changed.  This only happens with optional_all dependencies.
1703  */
1704 static void
1705 propagate_start(graph_vertex_t *v, void *arg)
1706 {
1707         restarter_error_t err = (restarter_error_t)arg;
1708 
1709         if (is_inst_bypassed(v))
1710                 return;
1711 
1712         switch (v->gv_type) {
1713         case GVT_INST:
1714                 /* Restarter */
1715                 if (inst_running(v)) {
1716                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1717                                 vertex_send_event(v,
1718                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1719                         }
1720                 } else {
1721                         graph_start_if_satisfied(v);
1722                 }
1723                 break;
1724 
1725         case GVT_GROUP:
1726                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1727                         graph_walk_dependents(v, propagate_stop,
1728                             (void *)RERR_RESTART);
1729                         break;
1730                 }
1731                 err = v->gv_restart;
1732                 /* FALLTHROUGH */
1733 
1734         case GVT_SVC:
1735                 graph_walk_dependents(v, propagate_start, (void *)err);
1736                 break;
1737 
1738         case GVT_FILE:
1739 #ifndef NDEBUG
1740                 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1741                     __FILE__, __LINE__);
1742 #endif
1743                 abort();
1744                 /* NOTREACHED */
1745 
1746         default:
1747 #ifndef NDEBUG
1748                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1749                     v->gv_type);
1750 #endif
1751                 abort();
1752         }
1753 }
1754 
1755 /*
1756  * propagate_stop()
1757  *
1758  * This function is used to propagate a stop event to the dependents of the
1759  * given vertex.  Any dependents that are online (or in degraded state) with
1760  * the restart_on property set to "restart" or "refresh" will be stopped as
1761  * their dependencies have just changed, propagate_start() will start them
1762  * again once their dependencies have been re-satisfied.
1763  */
1764 static void
1765 propagate_stop(graph_vertex_t *v, void *arg)
1766 {


1767         restarter_error_t err = (restarter_error_t)arg;
1768 
1769         if (is_inst_bypassed(v))
1770                 return;
1771 
1772         switch (v->gv_type) {
1773         case GVT_INST:
1774                 /* Restarter */
1775                 if (err > RERR_NONE && inst_running(v)) {
1776                         if (err == RERR_RESTART || err == RERR_REFRESH) {
1777                                 vertex_send_event(v,
1778                                     RESTARTER_EVENT_TYPE_STOP_RESET);
1779                         } else {
1780                                 vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1781                         }
1782                 }
1783                 break;
1784 
1785         case GVT_SVC:
1786                 graph_walk_dependents(v, propagate_stop, arg);
1787                 break;
1788 
1789         case GVT_FILE:
1790 #ifndef NDEBUG
1791                 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1792                     __FILE__, __LINE__);
1793 #endif
1794                 abort();
1795                 /* NOTREACHED */
1796 
1797         case GVT_GROUP:
1798                 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1799                         graph_walk_dependents(v, propagate_start,
1800                             (void *)RERR_NONE);
1801                         break;
1802                 }
1803 
1804                 if (err == RERR_NONE || err > v->gv_restart)
1805                         break;
1806 
1807                 graph_walk_dependents(v, propagate_stop, arg);












1808                 break;
1809 
1810         default:
1811 #ifndef NDEBUG
1812                 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1813                     v->gv_type);
1814 #endif
1815                 abort();
1816         }
1817 }
1818 
1819 void
1820 offline_vertex(graph_vertex_t *v)
1821 {
1822         scf_handle_t *h = libscf_handle_create_bound_loop();
1823         scf_instance_t *scf_inst = safe_scf_instance_create(h);
1824         scf_propertygroup_t *pg = safe_scf_pg_create(h);
1825         restarter_instance_state_t state, next_state;
1826         int r;
1827 


4346         graph_vertex_t *vv;
4347         graph_edge_t *e;
4348 
4349         assert(MUTEX_HELD(&dgraph_lock));
4350 
4351         for (e = uu_list_first(v->gv_dependents); e != NULL;
4352             e = uu_list_next(v->gv_dependents, e)) {
4353                 vv = e->ge_vertex;
4354                 if (vv->gv_type == GVT_INST) {
4355                         if ((vv->gv_flags & GV_CONFIGURED) == 0)
4356                                 continue;
4357 
4358                         if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4359                                 continue;
4360 
4361                         if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4362                             (vv->gv_state == RESTARTER_STATE_DEGRADED))
4363                                 return (B_FALSE);
4364                 } else {
4365                         /*
4366                          * Skip all excluded dependents and decide whether
4367                          * to offline the service based on the restart_on
4368                          * attribute.
4369                          */
4370                         if (is_depgrp_bypassed(vv))
4371                                 continue;
4372 
4373                         /*
4374                          * For dependency groups or service vertices, keep
4375                          * traversing to see if instances are running.
4376                          */
4377                         if (insubtree_dependents_down(vv) == B_FALSE)
4378                                 return (B_FALSE);
4379                 }
4380         }
4381 
4382         return (B_TRUE);
4383 }
4384 
4385 /*
4386  * Returns true only if none of this service's dependents are 'up' -- online,
4387  * degraded, or offline.
4388  */


4913  * Propagate a start, stop event, or a satisfiability event.
4914  *
4915  * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4916  * to direct dependents.  PROPAGATE_SAT propagates a start then walks the
4917  * full dependent graph to check for newly satisfied nodes.  This is
4918  * necessary for cases when non-direct dependents may be effected but direct
4919  * dependents may not (e.g. for optional_all evaluations, see the
4920  * propagate_satbility() comments).
4921  *
4922  * PROPAGATE_SAT should be used whenever a non-running service moves into
4923  * a state which can satisfy optional dependencies, like disabled or
4924  * maintenance.
4925  */
4926 void
4927 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4928     restarter_error_t rerr)
4929 {
4930         if (type == PROPAGATE_STOP) {
4931                 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4932         } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4933                 graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
4934 
4935                 if (type == PROPAGATE_SAT)
4936                         propagate_satbility(v);
4937         } else {
4938 #ifndef NDEBUG
4939                 uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4940                     __LINE__, type);
4941 #endif
4942                 abort();
4943         }
4944 }
4945 
4946 /*
4947  * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4948  * restarter.  If it is running, send _STOP.  Send _REMOVE_INSTANCE.  Delete
4949  * all property group dependencies, and the dependency on the restarter,
4950  * disposing of vertices as appropriate.  If other vertices depend on this
4951  * one, mark it unconfigured and return.  Otherwise remove the vertex.  Always
4952  * returns 0.
4953  */


4984                 err = uu_list_walk(v->gv_dependencies,
4985                     (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
4986                 assert(err == 0);
4987         }
4988 
4989         delete_instance_dependencies(v, B_TRUE);
4990 
4991         /*
4992          * Deleting an instance can both satisfy and unsatisfy dependencies,
4993          * depending on their type.  First propagate the stop as a RERR_RESTART
4994          * event -- deletion isn't a fault, just a normal stop.  This gives
4995          * dependent services the chance to do a clean shutdown.  Then, mark
4996          * the service as unconfigured and propagate the start event for the
4997          * optional_all dependencies that might have become satisfied.
4998          */
4999         graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
5000 
5001         v->gv_flags &= ~GV_CONFIGURED;
5002         v->gv_flags &= ~GV_DEATHROW;
5003 
5004         graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
5005         propagate_satbility(v);
5006 
5007         /*
5008          * If there are no (non-service) dependents, the vertex can be
5009          * completely removed.
5010          */
5011         if (v != milestone && v->gv_refs == 0 &&
5012             uu_list_numnodes(v->gv_dependents) == 1)
5013                 remove_inst_vertex(v);
5014 
5015         if (milestone > MILESTONE_NONE) {
5016                 void *cookie = NULL;
5017 
5018                 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5019                         v = e->ge_vertex;
5020 
5021                         if (vertex_unref(v) == VERTEX_INUSE)
5022                                 while (eval_subgraph(v, h) == ECONNABORTED)
5023                                         libscf_handle_rebind(h);
5024 


5400 
5401 /*
5402  * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5403  * to the instances that are supposed to go offline during an
5404  * administrative disable operation.
5405  */
5406 static int
5407 mark_subtree(graph_edge_t *e, void *arg)
5408 {
5409         graph_vertex_t *v;
5410         int r;
5411 
5412         v = e->ge_vertex;
5413 
5414         /* If it's already in the subgraph, skip. */
5415         if (v->gv_flags & GV_TOOFFLINE)
5416                 return (UU_WALK_NEXT);
5417 
5418         switch (v->gv_type) {
5419         case GVT_INST:
5420                 /* If the instance is already offline, skip it. */
5421                 if (!inst_running(v))
5422                         return (UU_WALK_NEXT);
5423 
5424                 v->gv_flags |= GV_TOOFFLINE;
5425                 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5426                 break;
5427         case GVT_GROUP:
5428                 /*
5429                  * Skip all excluded dependents and decide whether to offline
5430                  * the service based on the restart_on attribute.
5431                  */
5432                 if (is_depgrp_bypassed(v))
5433                         return (UU_WALK_NEXT);
5434                 break;
5435         }
5436 
5437         r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5438             0);
5439         assert(r == 0);
5440         return (UU_WALK_NEXT);
5441 }
5442 
5443 static int
5444 mark_subgraph(graph_edge_t *e, void *arg)
5445 {
5446         graph_vertex_t *v;
5447         int r;
5448         int optional = (int)arg;
5449 
5450         v = e->ge_vertex;