Print this page
7267 SMF is fast and loose with optional dependencies
Reviewed by: Dan McDonald <danmcd@omniti.com>
Reviewed by: Jerry Jelinek <jerry.jelinek@joyent.com>
Reviewed by: Albert Lee <trisk@omniti.com>
Reviewed by: Gordon Ross <gordon.w.ross@gmail.com>

Split Close
Expand all
Collapse all
          --- old/usr/src/cmd/svc/startd/graph.c
          +++ new/usr/src/cmd/svc/startd/graph.c
↓ open down ↓ 15 lines elided ↑ open up ↑
  16   16   * fields enclosed by brackets "[]" replaced with your own identifying
  17   17   * information: Portions Copyright [yyyy] [name of copyright owner]
  18   18   *
  19   19   * CDDL HEADER END
  20   20   */
  21   21  
  22   22  /*
  23   23   * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24   24   * Copyright (c) 2015, Syneto S.R.L. All rights reserved.
  25   25   * Copyright 2016 Toomas Soome <tsoome@me.com>
       26 + * Copyright 2016 RackTop Systems.
  26   27   */
  27   28  
  28   29  /*
  29   30   * graph.c - master restarter graph engine
  30   31   *
  31   32   *   The graph engine keeps a dependency graph of all service instances on the
  32   33   *   system, as recorded in the repository.  It decides when services should
  33   34   *   be brought up or down based on service states and dependencies and sends
  34   35   *   commands to restarters to effect any changes.  It also executes
  35   36   *   administrator commands sent by svcadm via the repository.
↓ open down ↓ 165 lines elided ↑ open up ↑
 201  202  /*
 202  203   * Services in these states are not considered 'down' by the
 203  204   * milestone/shutdown code.
 204  205   */
 205  206  #define up_state(state) ((state) == RESTARTER_STATE_ONLINE || \
 206  207          (state) == RESTARTER_STATE_DEGRADED || \
 207  208          (state) == RESTARTER_STATE_OFFLINE)
 208  209  
 209  210  #define is_depgrp_bypassed(v) ((v->gv_type == GVT_GROUP) && \
 210  211          ((v->gv_depgroup == DEPGRP_EXCLUDE_ALL) || \
 211      -        (v->gv_depgroup == DEPGRP_OPTIONAL_ALL) || \
 212  212          (v->gv_restart < RERR_RESTART)))
 213  213  
      214 +#define is_inst_bypassed(v) ((v->gv_type == GVT_INST) && \
      215 +        ((v->gv_flags & GV_TODISABLE) || \
      216 +        (v->gv_flags & GV_TOOFFLINE)))
      217 +
 214  218  static uu_list_pool_t *graph_edge_pool, *graph_vertex_pool;
 215  219  static uu_list_t *dgraph;
 216  220  static pthread_mutex_t dgraph_lock;
 217  221  
 218  222  /*
 219  223   * milestone indicates the current subgraph.  When NULL, it is the entire
 220  224   * graph.  When MILESTONE_NONE, it is the empty graph.  Otherwise, it is all
 221  225   * services on which the target vertex depends.
 222  226   */
 223  227  static graph_vertex_t *milestone = NULL;
↓ open down ↓ 1067 lines elided ↑ open up ↑
1291 1295  
1292 1296                  log_framework2(LOG_DEBUG, DEBUG_DEPENDENCIES,
1293 1297                      "require_any(%s): %s is unsatisfi%s.\n",
1294 1298                      groupv->gv_name, edge->ge_vertex->gv_name,
1295 1299                      s == 0 ? "ed" : "able");
1296 1300  
1297 1301                  if (satbility && s == 0)
1298 1302                          satisfiable = B_TRUE;
1299 1303          }
1300 1304  
1301      -        return (!satbility || satisfiable ? 0 : -1);
     1305 +        return ((!satbility || satisfiable) ? 0 : -1);
1302 1306  }
1303 1307  
1304 1308  /*
1305 1309   * An optional_all dependency only considers elements which are configured,
1306 1310   * enabled, and not in maintenance.  If any are unsatisfied, then the dependency
1307 1311   * is unsatisfied.
1308 1312   *
1309 1313   * Offline dependencies which are waiting for a dependency to come online are
1310 1314   * unsatisfied.  Offline dependences which cannot possibly come online
1311 1315   * (unsatisfiable) are always considered satisfied.
↓ open down ↓ 10 lines elided ↑ open up ↑
1322 1326          any_qualified = B_FALSE;
1323 1327          any_unsatisfied = B_FALSE;
1324 1328  
1325 1329          for (edge = uu_list_first(groupv->gv_dependencies);
1326 1330              edge != NULL;
1327 1331              edge = uu_list_next(groupv->gv_dependencies, edge)) {
1328 1332                  v = edge->ge_vertex;
1329 1333  
1330 1334                  switch (v->gv_type) {
1331 1335                  case GVT_INST:
1332      -                        /* Skip missing or disabled instances */
1333      -                        if ((v->gv_flags & (GV_CONFIGURED | GV_ENABLED)) !=
1334      -                            (GV_CONFIGURED | GV_ENABLED))
     1336 +                        /* Skip missing instances */
     1337 +                        if ((v->gv_flags & GV_CONFIGURED) == 0)
1335 1338                                  continue;
1336 1339  
1337 1340                          if (v->gv_state == RESTARTER_STATE_MAINT)
1338 1341                                  continue;
1339 1342  
1340      -                        if (v->gv_flags & GV_TOOFFLINE)
1341      -                                continue;
1342      -
1343 1343                          any_qualified = B_TRUE;
1344      -                        if (v->gv_state == RESTARTER_STATE_OFFLINE) {
     1344 +                        if (v->gv_state == RESTARTER_STATE_OFFLINE ||
     1345 +                            v->gv_state == RESTARTER_STATE_DISABLED) {
1345 1346                                  /*
1346      -                                 * For offline dependencies, treat unsatisfiable
1347      -                                 * as satisfied.
     1347 +                                 * For offline/disabled dependencies,
     1348 +                                 * treat unsatisfiable as satisfied.
1348 1349                                   */
1349 1350                                  i = dependency_satisfied(v, B_TRUE);
1350 1351                                  if (i == -1)
1351 1352                                          i = 1;
1352      -                        } else if (v->gv_state == RESTARTER_STATE_DISABLED) {
1353      -                                /*
1354      -                                 * The service is enabled, but hasn't
1355      -                                 * transitioned out of disabled yet.  Treat it
1356      -                                 * as unsatisfied (not unsatisfiable).
1357      -                                 */
1358      -                                i = 0;
1359 1353                          } else {
1360 1354                                  i = dependency_satisfied(v, satbility);
1361 1355                          }
1362 1356                          break;
1363 1357  
1364 1358                  case GVT_FILE:
1365 1359                          any_qualified = B_TRUE;
1366 1360                          i = dependency_satisfied(v, satbility);
1367 1361  
1368 1362                          break;
1369 1363  
1370 1364                  case GVT_SVC: {
1371      -                        boolean_t svc_any_qualified;
1372      -                        boolean_t svc_satisfied;
1373      -                        boolean_t svc_satisfiable;
1374      -                        graph_vertex_t *v2;
1375      -                        graph_edge_t *e2;
1376      -
1377      -                        svc_any_qualified = B_FALSE;
1378      -                        svc_satisfied = B_FALSE;
1379      -                        svc_satisfiable = B_FALSE;
1380      -
1381      -                        for (e2 = uu_list_first(v->gv_dependencies);
1382      -                            e2 != NULL;
1383      -                            e2 = uu_list_next(v->gv_dependencies, e2)) {
1384      -                                v2 = e2->ge_vertex;
1385      -                                assert(v2->gv_type == GVT_INST);
1386      -
1387      -                                if ((v2->gv_flags &
1388      -                                    (GV_CONFIGURED | GV_ENABLED)) !=
1389      -                                    (GV_CONFIGURED | GV_ENABLED))
1390      -                                        continue;
1391      -
1392      -                                if (v2->gv_state == RESTARTER_STATE_MAINT)
1393      -                                        continue;
1394      -
1395      -                                if (v2->gv_flags & GV_TOOFFLINE)
1396      -                                        continue;
1397      -
1398      -                                svc_any_qualified = B_TRUE;
1399      -
1400      -                                if (v2->gv_state == RESTARTER_STATE_OFFLINE) {
1401      -                                        /*
1402      -                                         * For offline dependencies, treat
1403      -                                         * unsatisfiable as satisfied.
1404      -                                         */
1405      -                                        i = dependency_satisfied(v2, B_TRUE);
1406      -                                        if (i == -1)
1407      -                                                i = 1;
1408      -                                } else if (v2->gv_state ==
1409      -                                    RESTARTER_STATE_DISABLED) {
1410      -                                        i = 0;
1411      -                                } else {
1412      -                                        i = dependency_satisfied(v2, satbility);
1413      -                                }
1414      -
1415      -                                if (i == 1) {
1416      -                                        svc_satisfied = B_TRUE;
1417      -                                        break;
1418      -                                }
1419      -                                if (i == 0)
1420      -                                        svc_satisfiable = B_TRUE;
1421      -                        }
1422      -
1423      -                        if (!svc_any_qualified)
1424      -                                continue;
1425 1365                          any_qualified = B_TRUE;
1426      -                        if (svc_satisfied) {
1427      -                                i = 1;
1428      -                        } else if (svc_satisfiable) {
1429      -                                i = 0;
1430      -                        } else {
1431      -                                i = -1;
1432      -                        }
     1366 +                        i = optional_all_satisfied(v, satbility);
     1367 +
1433 1368                          break;
1434 1369                  }
1435 1370  
1436 1371                  case GVT_GROUP:
1437 1372                  default:
1438 1373  #ifndef NDEBUG
1439 1374                          uu_warn("%s:%d: Unexpected vertex type %d.\n", __FILE__,
1440 1375                              __LINE__, v->gv_type);
1441 1376  #endif
1442 1377                          abort();
↓ open down ↓ 159 lines elided ↑ open up ↑
1602 1537                                  /*
1603 1538                                   * A dependency on an instance with GV_DEATHROW
1604 1539                                   * flag is always considered as satisfied.
1605 1540                                   */
1606 1541                                  return (1);
1607 1542                          }
1608 1543                          return (-1);
1609 1544                  }
1610 1545  
1611 1546                  /*
1612      -                 * Any vertex with the GV_TOOFFLINE flag set is guaranteed
1613      -                 * to have its dependencies unsatisfiable.
     1547 +                 * Vertices may be transitioning so we try to figure out if
     1548 +                 * the end state is likely to satisfy the dependency instead
     1549 +                 * of assuming the dependency is unsatisfied/unsatisfiable.
     1550 +                 *
     1551 +                 * Support for optional_all dependencies depends on us getting
     1552 +                 * this right because unsatisfiable dependencies are treated
     1553 +                 * as being satisfied.
1614 1554                   */
1615      -                if (v->gv_flags & GV_TOOFFLINE)
1616      -                        return (-1);
1617      -
1618 1555                  switch (v->gv_state) {
1619 1556                  case RESTARTER_STATE_ONLINE:
1620 1557                  case RESTARTER_STATE_DEGRADED:
     1558 +                        if (v->gv_flags & GV_TODISABLE)
     1559 +                                return (-1);
     1560 +                        if (v->gv_flags & GV_TOOFFLINE)
     1561 +                                return (0);
1621 1562                          return (1);
1622 1563  
1623 1564                  case RESTARTER_STATE_OFFLINE:
1624      -                        if (!satbility)
1625      -                                return (0);
     1565 +                        if (!satbility || v->gv_flags & GV_TODISABLE)
     1566 +                                return (satbility ? -1 : 0);
1626 1567                          return (instance_satisfied(v, satbility) != -1 ?
1627 1568                              0 : -1);
1628 1569  
1629 1570                  case RESTARTER_STATE_DISABLED:
     1571 +                        if (!satbility || !(v->gv_flags & GV_ENABLED))
     1572 +                                return (satbility ? -1 : 0);
     1573 +                        return (instance_satisfied(v, satbility) != -1 ?
     1574 +                            0 : -1);
     1575 +
1630 1576                  case RESTARTER_STATE_MAINT:
1631 1577                          return (-1);
1632 1578  
1633 1579                  case RESTARTER_STATE_UNINIT:
1634 1580                          return (0);
1635 1581  
1636 1582                  default:
1637 1583  #ifndef NDEBUG
1638 1584                          uu_warn("%s:%d: Unexpected vertex state %d.\n",
1639 1585                              __FILE__, __LINE__, v->gv_state);
↓ open down ↓ 82 lines elided ↑ open up ↑
1722 1668   * an optional_all dependency.  It's not possible for an optional_all dependency
1723 1669   * to change satisfiability without also coming online, in which case we get a
1724 1670   * start event and propagation continues naturally.  However, it does no harm to
1725 1671   * continue propagating satisfiability (as it is a relatively rare event), and
1726 1672   * keeps the walker code simple and generic.
1727 1673   */
1728 1674  /*ARGSUSED*/
1729 1675  static int
1730 1676  satbility_cb(graph_vertex_t *v, void *arg)
1731 1677  {
     1678 +        if (is_inst_bypassed(v))
     1679 +                return (UU_WALK_NEXT);
     1680 +
1732 1681          if (v->gv_type == GVT_INST)
1733 1682                  graph_start_if_satisfied(v);
1734 1683  
1735 1684          return (UU_WALK_NEXT);
1736 1685  }
1737 1686  
1738 1687  static void
1739 1688  propagate_satbility(graph_vertex_t *v)
1740 1689  {
1741 1690          graph_walk(v, WALK_DEPENDENTS, satbility_cb, NULL, NULL);
1742 1691  }
1743 1692  
1744 1693  static void propagate_stop(graph_vertex_t *, void *);
1745 1694  
1746      -/* ARGSUSED */
     1695 +/*
     1696 + * propagate_start()
     1697 + *
     1698 + * This function is used to propagate a start event to the dependents of the
     1699 + * given vertex.  Any dependents that are offline but have their dependencies
     1700 + * satisfied are started.  Any dependents that are online and have restart_on
     1701 + * set to "restart" or "refresh" are restarted because their dependencies have
     1702 + * just changed.  This only happens with optional_all dependencies.
     1703 + */
1747 1704  static void
1748 1705  propagate_start(graph_vertex_t *v, void *arg)
1749 1706  {
     1707 +        restarter_error_t err = (restarter_error_t)arg;
     1708 +
     1709 +        if (is_inst_bypassed(v))
     1710 +                return;
     1711 +
1750 1712          switch (v->gv_type) {
1751 1713          case GVT_INST:
1752      -                graph_start_if_satisfied(v);
     1714 +                /* Restarter */
     1715 +                if (inst_running(v)) {
     1716 +                        if (err == RERR_RESTART || err == RERR_REFRESH) {
     1717 +                                vertex_send_event(v,
     1718 +                                    RESTARTER_EVENT_TYPE_STOP_RESET);
     1719 +                        }
     1720 +                } else {
     1721 +                        graph_start_if_satisfied(v);
     1722 +                }
1753 1723                  break;
1754 1724  
1755 1725          case GVT_GROUP:
1756 1726                  if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1757 1727                          graph_walk_dependents(v, propagate_stop,
1758 1728                              (void *)RERR_RESTART);
1759 1729                          break;
1760 1730                  }
     1731 +                err = v->gv_restart;
1761 1732                  /* FALLTHROUGH */
1762 1733  
1763 1734          case GVT_SVC:
1764      -                graph_walk_dependents(v, propagate_start, NULL);
     1735 +                graph_walk_dependents(v, propagate_start, (void *)err);
1765 1736                  break;
1766 1737  
1767 1738          case GVT_FILE:
1768 1739  #ifndef NDEBUG
1769 1740                  uu_warn("%s:%d: propagate_start() encountered GVT_FILE.\n",
1770 1741                      __FILE__, __LINE__);
1771 1742  #endif
1772 1743                  abort();
1773 1744                  /* NOTREACHED */
1774 1745  
1775 1746          default:
1776 1747  #ifndef NDEBUG
1777 1748                  uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1778 1749                      v->gv_type);
1779 1750  #endif
1780 1751                  abort();
1781 1752          }
1782 1753  }
1783 1754  
     1755 +/*
     1756 + * propagate_stop()
     1757 + *
     1758 + * This function is used to propagate a stop event to the dependents of the
     1759 + * given vertex.  Any dependents that are online (or in degraded state) with
     1760 + * the restart_on property set to "restart" or "refresh" will be stopped as
     1761 + * their dependencies have just changed, propagate_start() will start them
     1762 + * again once their dependencies have been re-satisfied.
     1763 + */
1784 1764  static void
1785 1765  propagate_stop(graph_vertex_t *v, void *arg)
1786 1766  {
1787      -        graph_edge_t *e;
1788      -        graph_vertex_t *svc;
1789 1767          restarter_error_t err = (restarter_error_t)arg;
1790 1768  
     1769 +        if (is_inst_bypassed(v))
     1770 +                return;
     1771 +
1791 1772          switch (v->gv_type) {
1792 1773          case GVT_INST:
1793 1774                  /* Restarter */
1794 1775                  if (err > RERR_NONE && inst_running(v)) {
1795 1776                          if (err == RERR_RESTART || err == RERR_REFRESH) {
1796 1777                                  vertex_send_event(v,
1797 1778                                      RESTARTER_EVENT_TYPE_STOP_RESET);
1798 1779                          } else {
1799 1780                                  vertex_send_event(v, RESTARTER_EVENT_TYPE_STOP);
1800 1781                          }
↓ open down ↓ 7 lines elided ↑ open up ↑
1808 1789          case GVT_FILE:
1809 1790  #ifndef NDEBUG
1810 1791                  uu_warn("%s:%d: propagate_stop() encountered GVT_FILE.\n",
1811 1792                      __FILE__, __LINE__);
1812 1793  #endif
1813 1794                  abort();
1814 1795                  /* NOTREACHED */
1815 1796  
1816 1797          case GVT_GROUP:
1817 1798                  if (v->gv_depgroup == DEPGRP_EXCLUDE_ALL) {
1818      -                        graph_walk_dependents(v, propagate_start, NULL);
     1799 +                        graph_walk_dependents(v, propagate_start,
     1800 +                            (void *)RERR_NONE);
1819 1801                          break;
1820 1802                  }
1821 1803  
1822 1804                  if (err == RERR_NONE || err > v->gv_restart)
1823 1805                          break;
1824 1806  
1825      -                assert(uu_list_numnodes(v->gv_dependents) == 1);
1826      -                e = uu_list_first(v->gv_dependents);
1827      -                svc = e->ge_vertex;
1828      -
1829      -                if (inst_running(svc)) {
1830      -                        if (err == RERR_RESTART || err == RERR_REFRESH) {
1831      -                                vertex_send_event(svc,
1832      -                                    RESTARTER_EVENT_TYPE_STOP_RESET);
1833      -                        } else {
1834      -                                vertex_send_event(svc,
1835      -                                    RESTARTER_EVENT_TYPE_STOP);
1836      -                        }
1837      -                }
     1807 +                graph_walk_dependents(v, propagate_stop, arg);
1838 1808                  break;
1839 1809  
1840 1810          default:
1841 1811  #ifndef NDEBUG
1842 1812                  uu_warn("%s:%d: Unknown vertex type %d.\n", __FILE__, __LINE__,
1843 1813                      v->gv_type);
1844 1814  #endif
1845 1815                  abort();
1846 1816          }
1847 1817  }
↓ open down ↓ 2538 lines elided ↑ open up ↑
4386 4356                                  continue;
4387 4357  
4388 4358                          if ((vv->gv_flags & GV_TOOFFLINE) == 0)
4389 4359                                  continue;
4390 4360  
4391 4361                          if ((vv->gv_state == RESTARTER_STATE_ONLINE) ||
4392 4362                              (vv->gv_state == RESTARTER_STATE_DEGRADED))
4393 4363                                  return (B_FALSE);
4394 4364                  } else {
4395 4365                          /*
4396      -                         * Skip all excluded and optional_all dependencies
4397      -                         * and decide whether to offline the service based
4398      -                         * on restart_on attribute.
     4366 +                         * Skip all excluded dependents and decide whether
     4367 +                         * to offline the service based on the restart_on
     4368 +                         * attribute.
4399 4369                           */
4400 4370                          if (is_depgrp_bypassed(vv))
4401 4371                                  continue;
4402 4372  
4403 4373                          /*
4404 4374                           * For dependency groups or service vertices, keep
4405 4375                           * traversing to see if instances are running.
4406 4376                           */
4407 4377                          if (insubtree_dependents_down(vv) == B_FALSE)
4408 4378                                  return (B_FALSE);
↓ open down ↓ 544 lines elided ↑ open up ↑
4953 4923   * a state which can satisfy optional dependencies, like disabled or
4954 4924   * maintenance.
4955 4925   */
4956 4926  void
4957 4927  graph_transition_propagate(graph_vertex_t *v, propagate_event_t type,
4958 4928      restarter_error_t rerr)
4959 4929  {
4960 4930          if (type == PROPAGATE_STOP) {
4961 4931                  graph_walk_dependents(v, propagate_stop, (void *)rerr);
4962 4932          } else if (type == PROPAGATE_START || type == PROPAGATE_SAT) {
4963      -                graph_walk_dependents(v, propagate_start, NULL);
     4933 +                graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
4964 4934  
4965 4935                  if (type == PROPAGATE_SAT)
4966 4936                          propagate_satbility(v);
4967 4937          } else {
4968 4938  #ifndef NDEBUG
4969 4939                  uu_warn("%s:%d: Unexpected type value %d.\n",  __FILE__,
4970 4940                      __LINE__, type);
4971 4941  #endif
4972 4942                  abort();
4973 4943          }
↓ open down ↓ 50 lines elided ↑ open up ↑
5024 4994           * event -- deletion isn't a fault, just a normal stop.  This gives
5025 4995           * dependent services the chance to do a clean shutdown.  Then, mark
5026 4996           * the service as unconfigured and propagate the start event for the
5027 4997           * optional_all dependencies that might have become satisfied.
5028 4998           */
5029 4999          graph_walk_dependents(v, propagate_stop, (void *)RERR_RESTART);
5030 5000  
5031 5001          v->gv_flags &= ~GV_CONFIGURED;
5032 5002          v->gv_flags &= ~GV_DEATHROW;
5033 5003  
5034      -        graph_walk_dependents(v, propagate_start, NULL);
     5004 +        graph_walk_dependents(v, propagate_start, (void *)RERR_NONE);
5035 5005          propagate_satbility(v);
5036 5006  
5037 5007          /*
5038 5008           * If there are no (non-service) dependents, the vertex can be
5039 5009           * completely removed.
5040 5010           */
5041 5011          if (v != milestone && v->gv_refs == 0 &&
5042 5012              uu_list_numnodes(v->gv_dependents) == 1)
5043 5013                  remove_inst_vertex(v);
5044 5014  
↓ open down ↓ 395 lines elided ↑ open up ↑
5440 5410          int r;
5441 5411  
5442 5412          v = e->ge_vertex;
5443 5413  
5444 5414          /* If it's already in the subgraph, skip. */
5445 5415          if (v->gv_flags & GV_TOOFFLINE)
5446 5416                  return (UU_WALK_NEXT);
5447 5417  
5448 5418          switch (v->gv_type) {
5449 5419          case GVT_INST:
5450      -                /* If the instance is already disabled, skip it. */
5451      -                if (!(v->gv_flags & GV_ENABLED))
     5420 +                /* If the instance is already offline, skip it. */
     5421 +                if (!inst_running(v))
5452 5422                          return (UU_WALK_NEXT);
5453 5423  
5454 5424                  v->gv_flags |= GV_TOOFFLINE;
5455 5425                  log_framework(LOG_DEBUG, "%s added to subtree\n", v->gv_name);
5456 5426                  break;
5457 5427          case GVT_GROUP:
5458 5428                  /*
5459      -                 * Skip all excluded and optional_all dependencies and decide
5460      -                 * whether to offline the service based on restart_on attribute.
     5429 +                 * Skip all excluded dependents and decide whether to offline
     5430 +                 * the service based on the restart_on attribute.
5461 5431                   */
5462 5432                  if (is_depgrp_bypassed(v))
5463 5433                          return (UU_WALK_NEXT);
5464 5434                  break;
5465 5435          }
5466 5436  
5467 5437          r = uu_list_walk(v->gv_dependents, (uu_walk_fn_t *)mark_subtree, arg,
5468 5438              0);
5469 5439          assert(r == 0);
5470 5440          return (UU_WALK_NEXT);
↓ open down ↓ 1457 lines elided ↑ open up ↑
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX