6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
25 * Copyright 2016 Toomas Soome <tsoome@me.com>
26 */
27
28 /*
29 * graph.c - master restarter graph engine
30 *
31 * The graph engine keeps a dependency graph of all service instances on the
32 * system, as recorded in the repository. It decides when services should
33 * be brought up or down based on service states and dependencies and sends
34 * commands to restarters to effect any changes. It also executes
35 * administrator commands sent by svcadm via the repository.
36 *
37 * The graph is stored in uu_list_t *dgraph and its vertices are
38 * graph_vertex_t's, each of which has a name and an integer id unique to
39 * its name (see dict.c). A vertex's type attribute designates the type
40 * of object it represents: GVT_INST for service instances, GVT_SVC for
41 * service objects (since service instances may depend on another service,
42 * rather than service instance), GVT_FILE for files (which services may
43 * depend on), and GVT_GROUP for dependencies on multiple objects. GVT_GROUP
44 * vertices are necessary because dependency lists may have particular
45 * grouping types (require any, require all, optional, or exclude) and
191 * access to stn_global
192 */
193 int32_t stn_global;
194 /*
195 * info_events_all holds a flag to override notification parameters and send
196 * Information events for all state transitions.
197 * same about the need of a mutex here.
198 */
199 int info_events_all;
200
201 /*
202 * Services in these states are not considered 'down' by the
203 * milestone/shutdown code.
204 */
205 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
206 (state) == RESTARTER_STATE_DEGRADED || \
207 (state) == RESTARTER_STATE_OFFLINE)
208
209 #define is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
210 ((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \
211 (v->gv_depgroup == DEPGRP_OPTIONAL_ALL) || \
212 (v->gv_restart < RERR_RESTART)))
213
214 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
215 static uu_list_t *dgraph;
216 static pthread_mutex_t dgraph_lock;
217
218 /*
219 * milestone indicates the current subgraph. When NULL, it is the entire
220 * graph. When MILESTONE_NONE, it is the empty graph. Otherwise, it is all
221 * services on which the target vertex depends.
222 */
223 static graph_vertex_t *milestone = NULL;
224 static boolean_t initial_milestone_set = B_FALSE;
225 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
226
227 /* protected by dgraph_lock */
228 static boolean_t sulogin_thread_running = B_FALSE;
229 static boolean_t sulogin_running = B_FALSE;
230 static boolean_t console_login_ready = B_FALSE;
231
232 /* Number of services to come down to complete milestone transition. */
233 static uint_t non_subgraph_svcs;
1281
1282 satisfiable = B_FALSE;
1283
1284 for (edge = uu_list_first(groupv->gv_dependencies);
1285 edge != NULL;
1286 edge = uu_list_next(groupv->gv_dependencies, edge)) {
1287 s = dependency_satisfied(edge->ge_vertex, satbility);
1288
1289 if (s == 1)
1290 return (1);
1291
1292 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1293 "require_any(%s): %s is unsatisfi%s.\n",
1294 groupv->gv_name, edge->ge_vertex->gv_name,
1295 s == 0 ? "ed" : "able");
1296
1297 if (satbility && s == 0)
1298 satisfiable = B_TRUE;
1299 }
1300
1301 return (!satbility || satisfiable ? 0 : -1);
1302 }
1303
1304 /*
1305 * An optional_all dependency only considers elements which are configured,
1306 * enabled, and not in maintenance. If any are unsatisfied, then the dependency
1307 * is unsatisfied.
1308 *
1309 * Offline dependencies which are waiting for a dependency to come online are
1310 * unsatisfied. Offline dependences which cannot possibly come online
1311 * (unsatisfiable) are always considered satisfied.
1312 */
1313 static int
1314 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1315 {
1316 graph_edge_t *edge;
1317 graph_vertex_t *v;
1318 boolean_t any_qualified;
1319 boolean_t any_unsatisfied;
1320 int i;
1321
1322 any_qualified = B_FALSE;
1323 any_unsatisfied = B_FALSE;
1324
1325 for (edge = uu_list_first(groupv->gv_dependencies);
1326 edge != NULL;
1327 edge = uu_list_next(groupv->gv_dependencies, edge)) {
1328 v = edge->ge_vertex;
1329
1330 switch (v->gv_type) {
1331 case GVT_INST:
1332 /* Skip missing or disabled instances */
1333 if ((v->gv_flags & (GV_CONFIGURED | GV_ENABLED)) !=
1334 (GV_CONFIGURED | GV_ENABLED))
1335 continue;
1336
1337 if (v->gv_state == RESTARTER_STATE_MAINT)
1338 continue;
1339
1340 if (v->gv_flags & GV_TOOFFLINE)
1341 continue;
1342
1343 any_qualified = B_TRUE;
1344 if (v->gv_state == RESTARTER_STATE_OFFLINE) {
1345 /*
1346 * For offline dependencies, treat unsatisfiable
1347 * as satisfied.
1348 */
1349 i = dependency_satisfied(v, B_TRUE);
1350 if (i == -1)
1351 i = 1;
1352 } else if (v->gv_state == RESTARTER_STATE_DISABLED) {
1353 /*
1354 * The service is enabled, but hasn't
1355 * transitioned out of disabled yet. Treat it
1356 * as unsatisfied (not unsatisfiable).
1357 */
1358 i = 0;
1359 } else {
1360 i = dependency_satisfied(v, satbility);
1361 }
1362 break;
1363
1364 case GVT_FILE:
1365 any_qualified = B_TRUE;
1366 i = dependency_satisfied(v, satbility);
1367
1368 break;
1369
1370 case GVT_SVC: {
1371 boolean_t svc_any_qualified;
1372 boolean_t svc_satisfied;
1373 boolean_t svc_satisfiable;
1374 graph_vertex_t *v2;
1375 graph_edge_t *e2;
1376
1377 svc_any_qualified = B_FALSE;
1378 svc_satisfied = B_FALSE;
1379 svc_satisfiable = B_FALSE;
1380
1381 for (e2 = uu_list_first(v->gv_dependencies);
1382 e2 != NULL;
1383 e2 = uu_list_next(v->gv_dependencies, e2)) {
1384 v2 = e2->ge_vertex;
1385 assert(v2->gv_type == GVT_INST);
1386
1387 if ((v2->gv_flags &
1388 (GV_CONFIGURED | GV_ENABLED)) !=
1389 (GV_CONFIGURED | GV_ENABLED))
1390 continue;
1391
1392 if (v2->gv_state == RESTARTER_STATE_MAINT)
1393 continue;
1394
1395 if (v2->gv_flags & GV_TOOFFLINE)
1396 continue;
1397
1398 svc_any_qualified = B_TRUE;
1399
1400 if (v2->gv_state == RESTARTER_STATE_OFFLINE) {
1401 /*
1402 * For offline dependencies, treat
1403 * unsatisfiable as satisfied.
1404 */
1405 i = dependency_satisfied(v2, B_TRUE);
1406 if (i == -1)
1407 i = 1;
1408 } else if (v2->gv_state ==
1409 RESTARTER_STATE_DISABLED) {
1410 i = 0;
1411 } else {
1412 i = dependency_satisfied(v2, satbility);
1413 }
1414
1415 if (i == 1) {
1416 svc_satisfied = B_TRUE;
1417 break;
1418 }
1419 if (i == 0)
1420 svc_satisfiable = B_TRUE;
1421 }
1422
1423 if (!svc_any_qualified)
1424 continue;
1425 any_qualified = B_TRUE;
1426 if (svc_satisfied) {
1427 i = 1;
1428 } else if (svc_satisfiable) {
1429 i = 0;
1430 } else {
1431 i = -1;
1432 }
1433 break;
1434 }
1435
1436 case GVT_GROUP:
1437 default:
1438 #ifndef NDEBUG
1439 uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1440 __LINE__, v->gv_type);
1441 #endif
1442 abort();
1443 }
1444
1445 if (i == 1)
1446 continue;
1447
1448 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1449 "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1450 v->gv_name, i == 0 ? "ed" : "able");
1451
1452 if (!satbility)
1453 return (0);
1454 if (i == -1)
1455 return (-1);
1592 * Decide whether v can satisfy a dependency. v can either be a child of
1593 * a group vertex, or of an instance vertex.
1594 */
1595 static int
1596 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1597 {
1598 switch (v->gv_type) {
1599 case GVT_INST:
1600 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1601 if (v->gv_flags & GV_DEATHROW) {
1602 /*
1603 * A dependency on an instance with GV_DEATHROW
1604 * flag is always considered as satisfied.
1605 */
1606 return (1);
1607 }
1608 return (-1);
1609 }
1610
1611 /*
1612 * Any vertex with the GV_TOOFFLINE flag set is guaranteed
1613 * to have its dependencies unsatisfiable.
1614 */
1615 if (v->gv_flags & GV_TOOFFLINE)
1616 return (-1);
1617
1618 switch (v->gv_state) {
1619 case RESTARTER_STATE_ONLINE:
1620 case RESTARTER_STATE_DEGRADED:
1621 return (1);
1622
1623 case RESTARTER_STATE_OFFLINE:
1624 if (!satbility)
1625 return (0);
1626 return (instance_satisfied(v, satbility) != -1 ?
1627 0 : -1);
1628
1629 case RESTARTER_STATE_DISABLED:
1630 case RESTARTER_STATE_MAINT:
1631 return (-1);
1632
1633 case RESTARTER_STATE_UNINIT:
1634 return (0);
1635
1636 default:
1637 #ifndef NDEBUG
1638 uu_warn("%s:%d: Unexpected vertex state %d.\n",
1639 __FILE__, __LINE__, v->gv_state);
1640 #endif
1641 abort();
1642 /* NOTREACHED */
1643 }
1644
1645 case GVT_SVC:
1646 if (uu_list_numnodes(v->gv_dependencies) == 0)
1647 return (-1);
1648 return (require_any_satisfied(v, satbility));
1649
1712 * | A |--------------->| B |-------------->| C |
1713 * +-----+ +-----+ +-----+
1714 *
1715 * offline -> maintenance
1716 *
1717 * If C goes into maintenance, it's not enough simply to check B. Because A has
1718 * an optional dependency, what was previously an unsatisfiable situation is now
1719 * satisfied (B will never come online, even though its state hasn't changed).
1720 *
1721 * Note that it's not necessary to continue examining dependents after reaching
1722 * an optional_all dependency. It's not possible for an optional_all dependency
1723 * to change satisfiability without also coming online, in which case we get a
1724 * start event and propagation continues naturally. However, it does no harm to
1725 * continue propagating satisfiability (as it is a relatively rare event), and
1726 * keeps the walker code simple and generic.
1727 */
1728 /*ARGSUSED*/
1729 static int
1730 satbility_cb(graph_vertex_t *v, void *arg)
1731 {
1732 if (v->gv_type == GVT_INST)
1733 graph_start_if_satisfied(v);
1734
1735 return (UU_WALK_NEXT);
1736 }
1737
1738 static void
1739 propagate_satbility(graph_vertex_t *v)
1740 {
1741 graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1742 }
1743
1744 static void propagate_stop(graph_vertex_t *, void *);
1745
1746 /* ARGSUSED */
1747 static void
1748 propagate_start(graph_vertex_t *v, void *arg)
1749 {
1750 switch (v->gv_type) {
1751 case GVT_INST:
1752 graph_start_if_satisfied(v);
1753 break;
1754
1755 case GVT_GROUP:
1756 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1757 graph_walk_dependents(v, propagate_stop,
1758 (void *)RERR_RESTART);
1759 break;
1760 }
1761 /* FALLTHROUGH */
1762
1763 case GVT_SVC:
1764 graph_walk_dependents(v, propagate_start, NULL);
1765 break;
1766
1767 case GVT_FILE:
1768 #ifndef NDEBUG
1769 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1770 __FILE__, __LINE__);
1771 #endif
1772 abort();
1773 /* NOTREACHED */
1774
1775 default:
1776 #ifndef NDEBUG
1777 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1778 v->gv_type);
1779 #endif
1780 abort();
1781 }
1782 }
1783
1784 static void
1785 propagate_stop(graph_vertex_t *v, void *arg)
1786 {
1787 graph_edge_t *e;
1788 graph_vertex_t *svc;
1789 restarter_error_t err = (restarter_error_t)arg;
1790
1791 switch (v->gv_type) {
1792 case GVT_INST:
1793 /* Restarter */
1794 if (err > RERR_NONE && inst_running(v)) {
1795 if (err == RERR_RESTART || err == RERR_REFRESH) {
1796 vertex_send_event(v,
1797 RESTARTER_EVENT_TYPE_STOP_RESET);
1798 } else {
1799 vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1800 }
1801 }
1802 break;
1803
1804 case GVT_SVC:
1805 graph_walk_dependents(v, propagate_stop, arg);
1806 break;
1807
1808 case GVT_FILE:
1809 #ifndef NDEBUG
1810 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1811 __FILE__, __LINE__);
1812 #endif
1813 abort();
1814 /* NOTREACHED */
1815
1816 case GVT_GROUP:
1817 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1818 graph_walk_dependents(v, propagate_start, NULL);
1819 break;
1820 }
1821
1822 if (err == RERR_NONE || err > v->gv_restart)
1823 break;
1824
1825 assert(uu_list_numnodes(v->gv_dependents) == 1);
1826 e = uu_list_first(v->gv_dependents);
1827 svc = e->ge_vertex;
1828
1829 if (inst_running(svc)) {
1830 if (err == RERR_RESTART || err == RERR_REFRESH) {
1831 vertex_send_event(svc,
1832 RESTARTER_EVENT_TYPE_STOP_RESET);
1833 } else {
1834 vertex_send_event(svc,
1835 RESTARTER_EVENT_TYPE_STOP);
1836 }
1837 }
1838 break;
1839
1840 default:
1841 #ifndef NDEBUG
1842 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1843 v->gv_type);
1844 #endif
1845 abort();
1846 }
1847 }
1848
1849 void
1850 offline_vertex(graph_vertex_t *v)
1851 {
1852 scf_handle_t *h = libscf_handle_create_bound_loop();
1853 scf_instance_t *scf_inst = safe_scf_instance_create(h);
1854 scf_propertygroup_t *pg = safe_scf_pg_create(h);
1855 restarter_instance_state_t state, next_state;
1856 int r;
1857
4376 graph_vertex_t *vv;
4377 graph_edge_t *e;
4378
4379 assert(MUTEX_HELD(&dgraph_lock));
4380
4381 for (e = uu_list_first(v->gv_dependents); e != NULL;
4382 e = uu_list_next(v->gv_dependents, e)) {
4383 vv = e->ge_vertex;
4384 if (vv->gv_type == GVT_INST) {
4385 if ((vv->gv_flags & GV_CONFIGURED) == 0)
4386 continue;
4387
4388 if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4389 continue;
4390
4391 if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4392 (vv->gv_state == RESTARTER_STATE_DEGRADED))
4393 return (B_FALSE);
4394 } else {
4395 /*
4396 * Skip all excluded and optional_all dependencies
4397 * and decide whether to offline the service based
4398 * on restart_on attribute.
4399 */
4400 if (is_depgrp_bypassed(vv))
4401 continue;
4402
4403 /*
4404 * For dependency groups or service vertices, keep
4405 * traversing to see if instances are running.
4406 */
4407 if (insubtree_dependents_down(vv) == B_FALSE)
4408 return (B_FALSE);
4409 }
4410 }
4411
4412 return (B_TRUE);
4413 }
4414
4415 /*
4416 * Returns true only if none of this service's dependents are 'up' -- online,
4417 * degraded, or offline.
4418 */
4943 * Propagate a start, stop event, or a satisfiability event.
4944 *
4945 * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4946 * to direct dependents. PROPAGATE_SAT propagates a start then walks the
4947 * full dependent graph to check for newly satisfied nodes. This is
4948 * necessary for cases when non-direct dependents may be effected but direct
4949 * dependents may not (e.g. for optional_all evaluations, see the
4950 * propagate_satbility() comments).
4951 *
4952 * PROPAGATE_SAT should be used whenever a non-running service moves into
4953 * a state which can satisfy optional dependencies, like disabled or
4954 * maintenance.
4955 */
4956 void
4957 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4958 restarter_error_t rerr)
4959 {
4960 if (type == PROPAGATE_STOP) {
4961 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4962 } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4963 graph_walk_dependents(v, propagate_start, NULL);
4964
4965 if (type == PROPAGATE_SAT)
4966 propagate_satbility(v);
4967 } else {
4968 #ifndef NDEBUG
4969 uu_warn("%s:%d: Unexpected type value %d.\n", __FILE__,
4970 __LINE__, type);
4971 #endif
4972 abort();
4973 }
4974 }
4975
4976 /*
4977 * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4978 * restarter. If it is running, send _STOP. Send _REMOVE_INSTANCE. Delete
4979 * all property group dependencies, and the dependency on the restarter,
4980 * disposing of vertices as appropriate. If other vertices depend on this
4981 * one, mark it unconfigured and return. Otherwise remove the vertex. Always
4982 * returns 0.
4983 */
5014 err = uu_list_walk(v->gv_dependencies,
5015 (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
5016 assert(err == 0);
5017 }
5018
5019 delete_instance_dependencies(v, B_TRUE);
5020
5021 /*
5022 * Deleting an instance can both satisfy and unsatisfy dependencies,
5023 * depending on their type. First propagate the stop as a RERR_RESTART
5024 * event -- deletion isn't a fault, just a normal stop. This gives
5025 * dependent services the chance to do a clean shutdown. Then, mark
5026 * the service as unconfigured and propagate the start event for the
5027 * optional_all dependencies that might have become satisfied.
5028 */
5029 graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
5030
5031 v->gv_flags &= ~GV_CONFIGURED;
5032 v->gv_flags &= ~GV_DEATHROW;
5033
5034 graph_walk_dependents(v, propagate_start, NULL);
5035 propagate_satbility(v);
5036
5037 /*
5038 * If there are no (non-service) dependents, the vertex can be
5039 * completely removed.
5040 */
5041 if (v != milestone && v->gv_refs == 0 &&
5042 uu_list_numnodes(v->gv_dependents) == 1)
5043 remove_inst_vertex(v);
5044
5045 if (milestone > MILESTONE_NONE) {
5046 void *cookie = NULL;
5047
5048 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5049 v = e->ge_vertex;
5050
5051 if (vertex_unref(v) == VERTEX_INUSE)
5052 while (eval_subgraph(v, h) == ECONNABORTED)
5053 libscf_handle_rebind(h);
5054
5430
5431 /*
5432 * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5433 * to the instances that are supposed to go offline during an
5434 * administrative disable operation.
5435 */
5436 static int
5437 mark_subtree(graph_edge_t *e, void *arg)
5438 {
5439 graph_vertex_t *v;
5440 int r;
5441
5442 v = e->ge_vertex;
5443
5444 /* If it's already in the subgraph, skip. */
5445 if (v->gv_flags & GV_TOOFFLINE)
5446 return (UU_WALK_NEXT);
5447
5448 switch (v->gv_type) {
5449 case GVT_INST:
5450 /* If the instance is already disabled, skip it. */
5451 if (!(v->gv_flags & GV_ENABLED))
5452 return (UU_WALK_NEXT);
5453
5454 v->gv_flags |= GV_TOOFFLINE;
5455 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5456 break;
5457 case GVT_GROUP:
5458 /*
5459 * Skip all excluded and optional_all dependencies and decide
5460 * whether to offline the service based on restart_on attribute.
5461 */
5462 if (is_depgrp_bypassed(v))
5463 return (UU_WALK_NEXT);
5464 break;
5465 }
5466
5467 r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5468 0);
5469 assert(r == 0);
5470 return (UU_WALK_NEXT);
5471 }
5472
5473 static int
5474 mark_subgraph(graph_edge_t *e, void *arg)
5475 {
5476 graph_vertex_t *v;
5477 int r;
5478 int optional = (int)arg;
5479
5480 v = e->ge_vertex;
|
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
25 * Copyright 2016 Toomas Soome <tsoome@me.com>
26 * Copyright 2016 RackTop Systems.
27 */
28
29 /*
30 * graph.c - master restarter graph engine
31 *
32 * The graph engine keeps a dependency graph of all service instances on the
33 * system, as recorded in the repository. It decides when services should
34 * be brought up or down based on service states and dependencies and sends
35 * commands to restarters to effect any changes. It also executes
36 * administrator commands sent by svcadm via the repository.
37 *
38 * The graph is stored in uu_list_t *dgraph and its vertices are
39 * graph_vertex_t's, each of which has a name and an integer id unique to
40 * its name (see dict.c). A vertex's type attribute designates the type
41 * of object it represents: GVT_INST for service instances, GVT_SVC for
42 * service objects (since service instances may depend on another service,
43 * rather than service instance), GVT_FILE for files (which services may
44 * depend on), and GVT_GROUP for dependencies on multiple objects. GVT_GROUP
45 * vertices are necessary because dependency lists may have particular
46 * grouping types (require any, require all, optional, or exclude) and
192 * access to stn_global
193 */
194 int32_t stn_global;
195 /*
196 * info_events_all holds a flag to override notification parameters and send
197 * Information events for all state transitions.
198 * same about the need of a mutex here.
199 */
200 int info_events_all;
201
202 /*
203 * Services in these states are not considered 'down' by the
204 * milestone/shutdown code.
205 */
206 #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
207 (state) == RESTARTER_STATE_DEGRADED || \
208 (state) == RESTARTER_STATE_OFFLINE)
209
210 #define is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
211 ((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \
212 (v->gv_restart < RERR_RESTART)))
213
214 #define is_inst_bypassed(v) ((v->gv_type == GVT_INST) && \
215 ((v->gv_flags & GV_TODISABLE) || \
216 (v->gv_flags & GV_TOOFFLINE)))
217
218 static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
219 static uu_list_t *dgraph;
220 static pthread_mutex_t dgraph_lock;
221
222 /*
223 * milestone indicates the current subgraph. When NULL, it is the entire
224 * graph. When MILESTONE_NONE, it is the empty graph. Otherwise, it is all
225 * services on which the target vertex depends.
226 */
227 static graph_vertex_t *milestone = NULL;
228 static boolean_t initial_milestone_set = B_FALSE;
229 static pthread_cond_t initial_milestone_cv = PTHREAD_COND_INITIALIZER;
230
231 /* protected by dgraph_lock */
232 static boolean_t sulogin_thread_running = B_FALSE;
233 static boolean_t sulogin_running = B_FALSE;
234 static boolean_t console_login_ready = B_FALSE;
235
236 /* Number of services to come down to complete milestone transition. */
237 static uint_t non_subgraph_svcs;
1285
1286 satisfiable = B_FALSE;
1287
1288 for (edge = uu_list_first(groupv->gv_dependencies);
1289 edge != NULL;
1290 edge = uu_list_next(groupv->gv_dependencies, edge)) {
1291 s = dependency_satisfied(edge->ge_vertex, satbility);
1292
1293 if (s == 1)
1294 return (1);
1295
1296 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1297 "require_any(%s): %s is unsatisfi%s.\n",
1298 groupv->gv_name, edge->ge_vertex->gv_name,
1299 s == 0 ? "ed" : "able");
1300
1301 if (satbility && s == 0)
1302 satisfiable = B_TRUE;
1303 }
1304
1305 return ((!satbility || satisfiable) ? 0 : -1);
1306 }
1307
1308 /*
1309 * An optional_all dependency only considers elements which are configured,
1310 * enabled, and not in maintenance. If any are unsatisfied, then the dependency
1311 * is unsatisfied.
1312 *
1313 * Offline dependencies which are waiting for a dependency to come online are
1314 * unsatisfied. Offline dependences which cannot possibly come online
1315 * (unsatisfiable) are always considered satisfied.
1316 */
1317 static int
1318 optional_all_satisfied(graph_vertex_t *groupv, boolean_t satbility)
1319 {
1320 graph_edge_t *edge;
1321 graph_vertex_t *v;
1322 boolean_t any_qualified;
1323 boolean_t any_unsatisfied;
1324 int i;
1325
1326 any_qualified = B_FALSE;
1327 any_unsatisfied = B_FALSE;
1328
1329 for (edge = uu_list_first(groupv->gv_dependencies);
1330 edge != NULL;
1331 edge = uu_list_next(groupv->gv_dependencies, edge)) {
1332 v = edge->ge_vertex;
1333
1334 switch (v->gv_type) {
1335 case GVT_INST:
1336 /* Skip missing instances */
1337 if ((v->gv_flags & GV_CONFIGURED) == 0)
1338 continue;
1339
1340 if (v->gv_state == RESTARTER_STATE_MAINT)
1341 continue;
1342
1343 any_qualified = B_TRUE;
1344 if (v->gv_state == RESTARTER_STATE_OFFLINE ||
1345 v->gv_state == RESTARTER_STATE_DISABLED) {
1346 /*
1347 * For offline/disabled dependencies,
1348 * treat unsatisfiable as satisfied.
1349 */
1350 i = dependency_satisfied(v, B_TRUE);
1351 if (i == -1)
1352 i = 1;
1353 } else {
1354 i = dependency_satisfied(v, satbility);
1355 }
1356 break;
1357
1358 case GVT_FILE:
1359 any_qualified = B_TRUE;
1360 i = dependency_satisfied(v, satbility);
1361
1362 break;
1363
1364 case GVT_SVC: {
1365 any_qualified = B_TRUE;
1366 i = optional_all_satisfied(v, satbility);
1367
1368 break;
1369 }
1370
1371 case GVT_GROUP:
1372 default:
1373 #ifndef NDEBUG
1374 uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1375 __LINE__, v->gv_type);
1376 #endif
1377 abort();
1378 }
1379
1380 if (i == 1)
1381 continue;
1382
1383 log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1384 "optional_all(%s): %s is unsatisfi%s.\n", groupv->gv_name,
1385 v->gv_name, i == 0 ? "ed" : "able");
1386
1387 if (!satbility)
1388 return (0);
1389 if (i == -1)
1390 return (-1);
1527 * Decide whether v can satisfy a dependency. v can either be a child of
1528 * a group vertex, or of an instance vertex.
1529 */
1530 static int
1531 dependency_satisfied(graph_vertex_t *v, boolean_t satbility)
1532 {
1533 switch (v->gv_type) {
1534 case GVT_INST:
1535 if ((v->gv_flags & GV_CONFIGURED) == 0) {
1536 if (v->gv_flags & GV_DEATHROW) {
1537 /*
1538 * A dependency on an instance with GV_DEATHROW
1539 * flag is always considered as satisfied.
1540 */
1541 return (1);
1542 }
1543 return (-1);
1544 }
1545
1546 /*
1547 * Vertices may be transitioning so we try to figure out if
1548 * the end state is likely to satisfy the dependency instead
1549 * of assuming the dependency is unsatisfied/unsatisfiable.
1550 *
1551 * Support for optional_all dependencies depends on us getting
1552 * this right because unsatisfiable dependencies are treated
1553 * as being satisfied.
1554 */
1555 switch (v->gv_state) {
1556 case RESTARTER_STATE_ONLINE:
1557 case RESTARTER_STATE_DEGRADED:
1558 if (v->gv_flags & GV_TODISABLE)
1559 return (-1);
1560 if (v->gv_flags & GV_TOOFFLINE)
1561 return (0);
1562 return (1);
1563
1564 case RESTARTER_STATE_OFFLINE:
1565 if (!satbility || v->gv_flags & GV_TODISABLE)
1566 return (satbility ? -1 : 0);
1567 return (instance_satisfied(v, satbility) != -1 ?
1568 0 : -1);
1569
1570 case RESTARTER_STATE_DISABLED:
1571 if (!satbility || !(v->gv_flags & GV_ENABLED))
1572 return (satbility ? -1 : 0);
1573 return (instance_satisfied(v, satbility) != -1 ?
1574 0 : -1);
1575
1576 case RESTARTER_STATE_MAINT:
1577 return (-1);
1578
1579 case RESTARTER_STATE_UNINIT:
1580 return (0);
1581
1582 default:
1583 #ifndef NDEBUG
1584 uu_warn("%s:%d: Unexpected vertex state %d.\n",
1585 __FILE__, __LINE__, v->gv_state);
1586 #endif
1587 abort();
1588 /* NOTREACHED */
1589 }
1590
1591 case GVT_SVC:
1592 if (uu_list_numnodes(v->gv_dependencies) == 0)
1593 return (-1);
1594 return (require_any_satisfied(v, satbility));
1595
1658 * | A |--------------->| B |-------------->| C |
1659 * +-----+ +-----+ +-----+
1660 *
1661 * offline -> maintenance
1662 *
1663 * If C goes into maintenance, it's not enough simply to check B. Because A has
1664 * an optional dependency, what was previously an unsatisfiable situation is now
1665 * satisfied (B will never come online, even though its state hasn't changed).
1666 *
1667 * Note that it's not necessary to continue examining dependents after reaching
1668 * an optional_all dependency. It's not possible for an optional_all dependency
1669 * to change satisfiability without also coming online, in which case we get a
1670 * start event and propagation continues naturally. However, it does no harm to
1671 * continue propagating satisfiability (as it is a relatively rare event), and
1672 * keeps the walker code simple and generic.
1673 */
1674 /*ARGSUSED*/
1675 static int
1676 satbility_cb(graph_vertex_t *v, void *arg)
1677 {
1678 if (is_inst_bypassed(v))
1679 return (UU_WALK_NEXT);
1680
1681 if (v->gv_type == GVT_INST)
1682 graph_start_if_satisfied(v);
1683
1684 return (UU_WALK_NEXT);
1685 }
1686
1687 static void
1688 propagate_satbility(graph_vertex_t *v)
1689 {
1690 graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1691 }
1692
1693 static void propagate_stop(graph_vertex_t *, void *);
1694
1695 /*
1696 * propagate_start()
1697 *
1698 * This function is used to propagate a start event to the dependents of the
1699 * given vertex. Any dependents that are offline but have their dependencies
1700 * satisfied are started. Any dependents that are online and have restart_on
1701 * set to "restart" or "refresh" are restarted because their dependencies have
1702 * just changed. This only happens with optional_all dependencies.
1703 */
1704 static void
1705 propagate_start(graph_vertex_t *v, void *arg)
1706 {
1707 restarter_error_t err = (restarter_error_t)arg;
1708
1709 if (is_inst_bypassed(v))
1710 return;
1711
1712 switch (v->gv_type) {
1713 case GVT_INST:
1714 /* Restarter */
1715 if (inst_running(v)) {
1716 if (err == RERR_RESTART || err == RERR_REFRESH) {
1717 vertex_send_event(v,
1718 RESTARTER_EVENT_TYPE_STOP_RESET);
1719 }
1720 } else {
1721 graph_start_if_satisfied(v);
1722 }
1723 break;
1724
1725 case GVT_GROUP:
1726 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1727 graph_walk_dependents(v, propagate_stop,
1728 (void *)RERR_RESTART);
1729 break;
1730 }
1731 err = v->gv_restart;
1732 /* FALLTHROUGH */
1733
1734 case GVT_SVC:
1735 graph_walk_dependents(v, propagate_start, (void *)err);
1736 break;
1737
1738 case GVT_FILE:
1739 #ifndef NDEBUG
1740 uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1741 __FILE__, __LINE__);
1742 #endif
1743 abort();
1744 /* NOTREACHED */
1745
1746 default:
1747 #ifndef NDEBUG
1748 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1749 v->gv_type);
1750 #endif
1751 abort();
1752 }
1753 }
1754
1755 /*
1756 * propagate_stop()
1757 *
1758 * This function is used to propagate a stop event to the dependents of the
1759 * given vertex. Any dependents that are online (or in degraded state) with
1760 * the restart_on property set to "restart" or "refresh" will be stopped as
1761 * their dependencies have just changed, propagate_start() will start them
1762 * again once their dependencies have been re-satisfied.
1763 */
1764 static void
1765 propagate_stop(graph_vertex_t *v, void *arg)
1766 {
1767 restarter_error_t err = (restarter_error_t)arg;
1768
1769 if (is_inst_bypassed(v))
1770 return;
1771
1772 switch (v->gv_type) {
1773 case GVT_INST:
1774 /* Restarter */
1775 if (err > RERR_NONE && inst_running(v)) {
1776 if (err == RERR_RESTART || err == RERR_REFRESH) {
1777 vertex_send_event(v,
1778 RESTARTER_EVENT_TYPE_STOP_RESET);
1779 } else {
1780 vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1781 }
1782 }
1783 break;
1784
1785 case GVT_SVC:
1786 graph_walk_dependents(v, propagate_stop, arg);
1787 break;
1788
1789 case GVT_FILE:
1790 #ifndef NDEBUG
1791 uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1792 __FILE__, __LINE__);
1793 #endif
1794 abort();
1795 /* NOTREACHED */
1796
1797 case GVT_GROUP:
1798 if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1799 graph_walk_dependents(v, propagate_start,
1800 (void *)RERR_NONE);
1801 break;
1802 }
1803
1804 if (err == RERR_NONE || err > v->gv_restart)
1805 break;
1806
1807 graph_walk_dependents(v, propagate_stop, arg);
1808 break;
1809
1810 default:
1811 #ifndef NDEBUG
1812 uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1813 v->gv_type);
1814 #endif
1815 abort();
1816 }
1817 }
1818
1819 void
1820 offline_vertex(graph_vertex_t *v)
1821 {
1822 scf_handle_t *h = libscf_handle_create_bound_loop();
1823 scf_instance_t *scf_inst = safe_scf_instance_create(h);
1824 scf_propertygroup_t *pg = safe_scf_pg_create(h);
1825 restarter_instance_state_t state, next_state;
1826 int r;
1827
4346 graph_vertex_t *vv;
4347 graph_edge_t *e;
4348
4349 assert(MUTEX_HELD(&dgraph_lock));
4350
4351 for (e = uu_list_first(v->gv_dependents); e != NULL;
4352 e = uu_list_next(v->gv_dependents, e)) {
4353 vv = e->ge_vertex;
4354 if (vv->gv_type == GVT_INST) {
4355 if ((vv->gv_flags & GV_CONFIGURED) == 0)
4356 continue;
4357
4358 if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4359 continue;
4360
4361 if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4362 (vv->gv_state == RESTARTER_STATE_DEGRADED))
4363 return (B_FALSE);
4364 } else {
4365 /*
4366 * Skip all excluded dependents and decide whether
4367 * to offline the service based on the restart_on
4368 * attribute.
4369 */
4370 if (is_depgrp_bypassed(vv))
4371 continue;
4372
4373 /*
4374 * For dependency groups or service vertices, keep
4375 * traversing to see if instances are running.
4376 */
4377 if (insubtree_dependents_down(vv) == B_FALSE)
4378 return (B_FALSE);
4379 }
4380 }
4381
4382 return (B_TRUE);
4383 }
4384
4385 /*
4386 * Returns true only if none of this service's dependents are 'up' -- online,
4387 * degraded, or offline.
4388 */
4913 * Propagate a start, stop event, or a satisfiability event.
4914 *
4915 * PROPAGATE_START and PROPAGATE_STOP simply propagate the transition event
4916 * to direct dependents. PROPAGATE_SAT propagates a start then walks the
4917 * full dependent graph to check for newly satisfied nodes. This is
4918 * necessary for cases when non-direct dependents may be effected but direct
4919 * dependents may not (e.g. for optional_all evaluations, see the
4920 * propagate_satbility() comments).
4921 *
4922 * PROPAGATE_SAT should be used whenever a non-running service moves into
4923 * a state which can satisfy optional dependencies, like disabled or
4924 * maintenance.
4925 */
4926 void
4927 graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4928 restarter_error_t rerr)
4929 {
4930 if (type == PROPAGATE_STOP) {
4931 graph_walk_dependents(v, propagate_stop, (void *)rerr);
4932 } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4933 graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
4934
4935 if (type == PROPAGATE_SAT)
4936 propagate_satbility(v);
4937 } else {
4938 #ifndef NDEBUG
4939 uu_warn("%s:%d: Unexpected type value %d.\n", __FILE__,
4940 __LINE__, type);
4941 #endif
4942 abort();
4943 }
4944 }
4945
4946 /*
4947 * If a vertex for fmri exists and it is enabled, send _DISABLE to the
4948 * restarter. If it is running, send _STOP. Send _REMOVE_INSTANCE. Delete
4949 * all property group dependencies, and the dependency on the restarter,
4950 * disposing of vertices as appropriate. If other vertices depend on this
4951 * one, mark it unconfigured and return. Otherwise remove the vertex. Always
4952 * returns 0.
4953 */
4984 err = uu_list_walk(v->gv_dependencies,
4985 (uu_walk_fn_t *)append_svcs_or_insts, old_deps, 0);
4986 assert(err == 0);
4987 }
4988
4989 delete_instance_dependencies(v, B_TRUE);
4990
4991 /*
4992 * Deleting an instance can both satisfy and unsatisfy dependencies,
4993 * depending on their type. First propagate the stop as a RERR_RESTART
4994 * event -- deletion isn't a fault, just a normal stop. This gives
4995 * dependent services the chance to do a clean shutdown. Then, mark
4996 * the service as unconfigured and propagate the start event for the
4997 * optional_all dependencies that might have become satisfied.
4998 */
4999 graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
5000
5001 v->gv_flags &= ~GV_CONFIGURED;
5002 v->gv_flags &= ~GV_DEATHROW;
5003
5004 graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
5005 propagate_satbility(v);
5006
5007 /*
5008 * If there are no (non-service) dependents, the vertex can be
5009 * completely removed.
5010 */
5011 if (v != milestone && v->gv_refs == 0 &&
5012 uu_list_numnodes(v->gv_dependents) == 1)
5013 remove_inst_vertex(v);
5014
5015 if (milestone > MILESTONE_NONE) {
5016 void *cookie = NULL;
5017
5018 while ((e = uu_list_teardown(old_deps, &cookie)) != NULL) {
5019 v = e->ge_vertex;
5020
5021 if (vertex_unref(v) == VERTEX_INUSE)
5022 while (eval_subgraph(v, h) == ECONNABORTED)
5023 libscf_handle_rebind(h);
5024
5400
5401 /*
5402 * mark_subtree walks the dependents and add the GV_TOOFFLINE flag
5403 * to the instances that are supposed to go offline during an
5404 * administrative disable operation.
5405 */
5406 static int
5407 mark_subtree(graph_edge_t *e, void *arg)
5408 {
5409 graph_vertex_t *v;
5410 int r;
5411
5412 v = e->ge_vertex;
5413
5414 /* If it's already in the subgraph, skip. */
5415 if (v->gv_flags & GV_TOOFFLINE)
5416 return (UU_WALK_NEXT);
5417
5418 switch (v->gv_type) {
5419 case GVT_INST:
5420 /* If the instance is already offline, skip it. */
5421 if (!inst_running(v))
5422 return (UU_WALK_NEXT);
5423
5424 v->gv_flags |= GV_TOOFFLINE;
5425 log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5426 break;
5427 case GVT_GROUP:
5428 /*
5429 * Skip all excluded dependents and decide whether to offline
5430 * the service based on the restart_on attribute.
5431 */
5432 if (is_depgrp_bypassed(v))
5433 return (UU_WALK_NEXT);
5434 break;
5435 }
5436
5437 r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5438 0);
5439 assert(r == 0);
5440 return (UU_WALK_NEXT);
5441 }
5442
5443 static int
5444 mark_subgraph(graph_edge_t *e, void *arg)
5445 {
5446 graph_vertex_t *v;
5447 int r;
5448 int optional = (int)arg;
5449
5450 v = e->ge_vertex;
|