1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2018 Joyent, Inc.
  25  */
  26 
  27 /*
  28  * restarter.c - service manipulation
  29  *
  30  * This component manages services whose restarter is svc.startd, the standard
  31  * restarter.  It translates restarter protocol events from the graph engine
  32  * into actions on processes, as a delegated restarter would do.
  33  *
  34  * The master restarter manages a number of always-running threads:
  35  *   - restarter event thread: events from the graph engine
  36  *   - timeout thread: thread to fire queued timeouts
  37  *   - contract thread: thread to handle contract events
  38  *   - wait thread: thread to handle wait-based services
  39  *
  40  * The other threads are created as-needed:
  41  *   - per-instance method threads
  42  *   - per-instance event processing threads
  43  *
  44  * The interaction of all threads must result in the following conditions
  45  * being satisfied (on a per-instance basis):
  46  *   - restarter events must be processed in order
  47  *   - method execution must be serialized
  48  *   - instance delete must be held until outstanding methods are complete
  49  *   - contract events shouldn't be processed while a method is running
  50  *   - timeouts should fire even when a method is running
  51  *
  52  * Service instances are represented by restarter_inst_t's and are kept in the
  53  * instance_list list.
  54  *
  55  * Service States
  56  *   The current state of a service instance is kept in
  57  *   restarter_inst_t->ri_i.i_state.  If transition to a new state could take
  58  *   some time, then before we effect the transition we set
  59  *   restarter_inst_t->ri_i.i_next_state to the target state, and afterwards we
  60  *   rotate i_next_state to i_state and set i_next_state to
  61  *   RESTARTER_STATE_NONE.  So usually i_next_state is _NONE when ri_lock is not
  62  *   held.  The exception is when we launch methods, which are done with
  63  *   a separate thread.  To keep any other threads from grabbing ri_lock before
  64  *   method_thread() does, we set ri_method_thread to the thread id of the
  65  *   method thread, and when it is nonzero any thread with a different thread id
  66  *   waits on ri_method_cv.
  67  *
  68  * Method execution is serialized by blocking on ri_method_cv in
  69  * inst_lookup_by_id() and waiting for a 0 value of ri_method_thread.  This
  70  * also prevents the instance structure from being deleted until all
  71  * outstanding operations such as method_thread() have finished.
  72  *
  73  * Lock ordering:
  74  *
  75  * dgraph_lock [can be held when taking:]
  76  *   utmpx_lock
  77  *   dictionary->dict_lock
  78  *   st->st_load_lock
  79  *   wait_info_lock
  80  *   ru->restarter_update_lock
  81  *     restarter_queue->rpeq_lock
  82  *   instance_list.ril_lock
  83  *     inst->ri_lock
  84  *   st->st_configd_live_lock
  85  *
  86  * instance_list.ril_lock
  87  *   graph_queue->gpeq_lock
  88  *   gu->gu_lock
  89  *   st->st_configd_live_lock
  90  *   dictionary->dict_lock
  91  *   inst->ri_lock
  92  *     graph_queue->gpeq_lock
  93  *     gu->gu_lock
  94  *     tu->tu_lock
  95  *     tq->tq_lock
  96  *     inst->ri_queue_lock
  97  *       wait_info_lock
  98  *       bp->cb_lock
  99  *     utmpx_lock
 100  *
 101  * single_user_thread_lock
 102  *   wait_info_lock
 103  *   utmpx_lock
 104  *
 105  * gu_freeze_lock
 106  *
 107  * logbuf_mutex nests inside pretty much everything.
 108  */
 109 
 110 #include <sys/contract/process.h>
 111 #include <sys/ctfs.h>
 112 #include <sys/stat.h>
 113 #include <sys/time.h>
 114 #include <sys/types.h>
 115 #include <sys/uio.h>
 116 #include <sys/wait.h>
 117 #include <assert.h>
 118 #include <errno.h>
 119 #include <fcntl.h>
 120 #include <libcontract.h>
 121 #include <libcontract_priv.h>
 122 #include <libintl.h>
 123 #include <librestart.h>
 124 #include <librestart_priv.h>
 125 #include <libuutil.h>
 126 #include <limits.h>
 127 #include <poll.h>
 128 #include <port.h>
 129 #include <pthread.h>
 130 #include <stdarg.h>
 131 #include <stdio.h>
 132 #include <strings.h>
 133 #include <unistd.h>
 134 
 135 #include "startd.h"
 136 #include "protocol.h"
 137 
 138 static uu_list_pool_t *restarter_instance_pool;
 139 static restarter_instance_list_t instance_list;
 140 
 141 static uu_list_pool_t *restarter_queue_pool;
 142 
 143 #define WT_SVC_ERR_THROTTLE     1       /* 1 sec delay for erroring wait svc */
 144 
 145 /*
 146  * Function used to reset the restart times for an instance, when
 147  * an administrative task comes along and essentially makes the times
 148  * in this array ineffective.
 149  */
 150 static void
 151 reset_start_times(restarter_inst_t *inst)
 152 {
 153         inst->ri_start_index = 0;
 154         bzero(inst->ri_start_time, sizeof (inst->ri_start_time));
 155 }
 156 
 157 /*ARGSUSED*/
 158 static int
 159 restarter_instance_compare(const void *lc_arg, const void *rc_arg,
 160     void *private)
 161 {
 162         int lc_id = ((const restarter_inst_t *)lc_arg)->ri_id;
 163         int rc_id = *(int *)rc_arg;
 164 
 165         if (lc_id > rc_id)
 166                 return (1);
 167         if (lc_id < rc_id)
 168                 return (-1);
 169         return (0);
 170 }
 171 
 172 static restarter_inst_t *
 173 inst_lookup_by_name(const char *name)
 174 {
 175         int id;
 176 
 177         id = dict_lookup_byname(name);
 178         if (id == -1)
 179                 return (NULL);
 180 
 181         return (inst_lookup_by_id(id));
 182 }
 183 
 184 restarter_inst_t *
 185 inst_lookup_by_id(int id)
 186 {
 187         restarter_inst_t *inst;
 188 
 189         MUTEX_LOCK(&instance_list.ril_lock);
 190         inst = uu_list_find(instance_list.ril_instance_list, &id, NULL, NULL);
 191         if (inst != NULL)
 192                 MUTEX_LOCK(&inst->ri_lock);
 193         MUTEX_UNLOCK(&instance_list.ril_lock);
 194 
 195         if (inst != NULL) {
 196                 while (inst->ri_method_thread != 0 &&
 197                     !pthread_equal(inst->ri_method_thread, pthread_self())) {
 198                         ++inst->ri_method_waiters;
 199                         (void) pthread_cond_wait(&inst->ri_method_cv,
 200                             &inst->ri_lock);
 201                         assert(inst->ri_method_waiters > 0);
 202                         --inst->ri_method_waiters;
 203                 }
 204         }
 205 
 206         return (inst);
 207 }
 208 
 209 static restarter_inst_t *
 210 inst_lookup_queue(const char *name)
 211 {
 212         int id;
 213         restarter_inst_t *inst;
 214 
 215         id = dict_lookup_byname(name);
 216         if (id == -1)
 217                 return (NULL);
 218 
 219         MUTEX_LOCK(&instance_list.ril_lock);
 220         inst = uu_list_find(instance_list.ril_instance_list, &id, NULL, NULL);
 221         if (inst != NULL)
 222                 MUTEX_LOCK(&inst->ri_queue_lock);
 223         MUTEX_UNLOCK(&instance_list.ril_lock);
 224 
 225         return (inst);
 226 }
 227 
 228 const char *
 229 service_style(int flags)
 230 {
 231         switch (flags & RINST_STYLE_MASK) {
 232         case RINST_CONTRACT:    return ("contract");
 233         case RINST_TRANSIENT:   return ("transient");
 234         case RINST_WAIT:        return ("wait");
 235 
 236         default:
 237 #ifndef NDEBUG
 238                 uu_warn("%s:%d: Bad flags 0x%x.\n", __FILE__, __LINE__, flags);
 239 #endif
 240                 abort();
 241                 /* NOTREACHED */
 242         }
 243 }
 244 
 245 /*
 246  * Fails with ECONNABORTED or ECANCELED.
 247  */
 248 static int
 249 check_contract(restarter_inst_t *inst, boolean_t primary,
 250     scf_instance_t *scf_inst)
 251 {
 252         ctid_t *ctidp;
 253         int fd, r;
 254 
 255         ctidp = primary ? &inst->ri_i.i_primary_ctid :
 256             &inst->ri_i.i_transient_ctid;
 257 
 258         assert(*ctidp >= 1);
 259 
 260         fd = contract_open(*ctidp, NULL, "status", O_RDONLY);
 261         if (fd >= 0) {
 262                 r = close(fd);
 263                 assert(r == 0);
 264                 return (0);
 265         }
 266 
 267         r = restarter_remove_contract(scf_inst, *ctidp, primary ?
 268             RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
 269         switch (r) {
 270         case 0:
 271         case ECONNABORTED:
 272         case ECANCELED:
 273                 *ctidp = 0;
 274                 return (r);
 275 
 276         case ENOMEM:
 277                 uu_die("Out of memory\n");
 278                 /* NOTREACHED */
 279 
 280         case EPERM:
 281                 uu_die("Insufficient privilege.\n");
 282                 /* NOTREACHED */
 283 
 284         case EACCES:
 285                 uu_die("Repository backend access denied.\n");
 286                 /* NOTREACHED */
 287 
 288         case EROFS:
 289                 log_error(LOG_INFO, "Could not remove unusable contract id %ld "
 290                     "for %s from repository.\n", *ctidp, inst->ri_i.i_fmri);
 291                 return (0);
 292 
 293         case EINVAL:
 294         case EBADF:
 295         default:
 296                 assert(0);
 297                 abort();
 298                 /* NOTREACHED */
 299         }
 300 }
 301 
 302 static int stop_instance(scf_handle_t *, restarter_inst_t *, stop_cause_t);
 303 
 304 /*
 305  * int restarter_insert_inst(scf_handle_t *, char *)
 306  *   If the inst is already in the restarter list, return its id.  If the inst
 307  *   is not in the restarter list, initialize a restarter_inst_t, initialize its
 308  *   states, insert it into the list, and return 0.
 309  *
 310  *   Fails with
 311  *     ENOENT - name is not in the repository
 312  */
 313 static int
 314 restarter_insert_inst(scf_handle_t *h, const char *name)
 315 {
 316         int id, r;
 317         restarter_inst_t *inst;
 318         uu_list_index_t idx;
 319         scf_service_t *scf_svc;
 320         scf_instance_t *scf_inst;
 321         scf_snapshot_t *snap = NULL;
 322         scf_propertygroup_t *pg;
 323         char *svc_name, *inst_name;
 324         char logfilebuf[PATH_MAX];
 325         char *c;
 326         boolean_t do_commit_states;
 327         restarter_instance_state_t state, next_state;
 328         protocol_states_t *ps;
 329         pid_t start_pid;
 330         restarter_str_t reason = restarter_str_insert_in_graph;
 331 
 332         MUTEX_LOCK(&instance_list.ril_lock);
 333 
 334         /*
 335          * We don't use inst_lookup_by_name() here because we want the lookup
 336          * & insert to be atomic.
 337          */
 338         id = dict_lookup_byname(name);
 339         if (id != -1) {
 340                 inst = uu_list_find(instance_list.ril_instance_list, &id, NULL,
 341                     &idx);
 342                 if (inst != NULL) {
 343                         MUTEX_UNLOCK(&instance_list.ril_lock);
 344                         return (0);
 345                 }
 346         }
 347 
 348         /* Allocate an instance */
 349         inst = startd_zalloc(sizeof (restarter_inst_t));
 350         inst->ri_utmpx_prefix = startd_alloc(max_scf_value_size);
 351         inst->ri_utmpx_prefix[0] = '\0';
 352 
 353         inst->ri_i.i_fmri = startd_alloc(strlen(name) + 1);
 354         (void) strcpy((char *)inst->ri_i.i_fmri, name);
 355 
 356         inst->ri_queue = startd_list_create(restarter_queue_pool, inst, 0);
 357 
 358         /*
 359          * id shouldn't be -1 since we use the same dictionary as graph.c, but
 360          * just in case.
 361          */
 362         inst->ri_id = (id != -1 ? id : dict_insert(name));
 363 
 364         special_online_hooks_get(name, &inst->ri_pre_online_hook,
 365             &inst->ri_post_online_hook, &inst->ri_post_offline_hook);
 366 
 367         scf_svc = safe_scf_service_create(h);
 368         scf_inst = safe_scf_instance_create(h);
 369         pg = safe_scf_pg_create(h);
 370         svc_name = startd_alloc(max_scf_name_size);
 371         inst_name = startd_alloc(max_scf_name_size);
 372 
 373 rep_retry:
 374         if (snap != NULL)
 375                 scf_snapshot_destroy(snap);
 376         if (inst->ri_logstem != NULL)
 377                 startd_free(inst->ri_logstem, PATH_MAX);
 378         if (inst->ri_common_name != NULL)
 379                 free(inst->ri_common_name);
 380         if (inst->ri_C_common_name != NULL)
 381                 free(inst->ri_C_common_name);
 382         snap = NULL;
 383         inst->ri_logstem = NULL;
 384         inst->ri_common_name = NULL;
 385         inst->ri_C_common_name = NULL;
 386 
 387         if (scf_handle_decode_fmri(h, name, NULL, scf_svc, scf_inst, NULL,
 388             NULL, SCF_DECODE_FMRI_EXACT) != 0) {
 389                 switch (scf_error()) {
 390                 case SCF_ERROR_CONNECTION_BROKEN:
 391                         libscf_handle_rebind(h);
 392                         goto rep_retry;
 393 
 394                 case SCF_ERROR_NOT_FOUND:
 395                         goto deleted;
 396                 }
 397 
 398                 uu_die("Can't decode FMRI %s: %s\n", name,
 399                     scf_strerror(scf_error()));
 400         }
 401 
 402         /*
 403          * If there's no running snapshot, then we execute using the editing
 404          * snapshot.  Pending snapshots will be taken later.
 405          */
 406         snap = libscf_get_running_snapshot(scf_inst);
 407 
 408         if ((scf_service_get_name(scf_svc, svc_name, max_scf_name_size) < 0) ||
 409             (scf_instance_get_name(scf_inst, inst_name, max_scf_name_size) <
 410             0)) {
 411                 switch (scf_error()) {
 412                 case SCF_ERROR_NOT_SET:
 413                         break;
 414 
 415                 case SCF_ERROR_CONNECTION_BROKEN:
 416                         libscf_handle_rebind(h);
 417                         goto rep_retry;
 418 
 419                 default:
 420                         assert(0);
 421                         abort();
 422                 }
 423 
 424                 goto deleted;
 425         }
 426 
 427         (void) snprintf(logfilebuf, PATH_MAX, "%s:%s", svc_name, inst_name);
 428         for (c = logfilebuf; *c != '\0'; c++)
 429                 if (*c == '/')
 430                         *c = '-';
 431 
 432         inst->ri_logstem = startd_alloc(PATH_MAX);
 433         (void) snprintf(inst->ri_logstem, PATH_MAX, "%s%s", logfilebuf,
 434             LOG_SUFFIX);
 435 
 436         /*
 437          * If the restarter group is missing, use uninit/none.  Otherwise,
 438          * we're probably being restarted & don't want to mess up the states
 439          * that are there.
 440          */
 441         state = RESTARTER_STATE_UNINIT;
 442         next_state = RESTARTER_STATE_NONE;
 443 
 444         r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
 445         if (r != 0) {
 446                 switch (scf_error()) {
 447                 case SCF_ERROR_CONNECTION_BROKEN:
 448                         libscf_handle_rebind(h);
 449                         goto rep_retry;
 450 
 451                 case SCF_ERROR_NOT_SET:
 452                         goto deleted;
 453 
 454                 case SCF_ERROR_NOT_FOUND:
 455                         /*
 456                          * This shouldn't happen since the graph engine should
 457                          * have initialized the state to uninitialized/none if
 458                          * there was no restarter pg.  In case somebody
 459                          * deleted it, though....
 460                          */
 461                         do_commit_states = B_TRUE;
 462                         break;
 463 
 464                 default:
 465                         assert(0);
 466                         abort();
 467                 }
 468         } else {
 469                 r = libscf_read_states(pg, &state, &next_state);
 470                 if (r != 0) {
 471                         do_commit_states = B_TRUE;
 472                 } else {
 473                         if (next_state != RESTARTER_STATE_NONE) {
 474                                 /*
 475                                  * Force next_state to _NONE since we
 476                                  * don't look for method processes.
 477                                  */
 478                                 next_state = RESTARTER_STATE_NONE;
 479                                 do_commit_states = B_TRUE;
 480                         } else {
 481                                 /*
 482                                  * The reason for transition will depend on
 483                                  * state.
 484                                  */
 485                                 if (st->st_initial == 0)
 486                                         reason = restarter_str_startd_restart;
 487                                 else if (state == RESTARTER_STATE_MAINT)
 488                                         reason = restarter_str_bad_repo_state;
 489                                 /*
 490                                  * Inform the restarter of our state without
 491                                  * changing the STIME in the repository.
 492                                  */
 493                                 ps = startd_alloc(sizeof (*ps));
 494                                 inst->ri_i.i_state = ps->ps_state = state;
 495                                 inst->ri_i.i_next_state = ps->ps_state_next =
 496                                     next_state;
 497                                 ps->ps_reason = reason;
 498 
 499                                 graph_protocol_send_event(inst->ri_i.i_fmri,
 500                                     GRAPH_UPDATE_STATE_CHANGE, ps);
 501 
 502                                 do_commit_states = B_FALSE;
 503                         }
 504                 }
 505         }
 506 
 507         switch (libscf_get_startd_properties(scf_inst, snap, &inst->ri_flags,
 508             &inst->ri_utmpx_prefix)) {
 509         case 0:
 510                 break;
 511 
 512         case ECONNABORTED:
 513                 libscf_handle_rebind(h);
 514                 goto rep_retry;
 515 
 516         case ECANCELED:
 517                 goto deleted;
 518 
 519         case ENOENT:
 520                 /*
 521                  * This is odd, because the graph engine should have required
 522                  * the general property group.  So we'll just use default
 523                  * flags in anticipation of the graph engine sending us
 524                  * REMOVE_INSTANCE when it finds out that the general property
 525                  * group has been deleted.
 526                  */
 527                 inst->ri_flags = RINST_CONTRACT;
 528                 break;
 529 
 530         default:
 531                 assert(0);
 532                 abort();
 533         }
 534 
 535         r = libscf_get_template_values(scf_inst, snap,
 536             &inst->ri_common_name, &inst->ri_C_common_name);
 537 
 538         /*
 539          * Copy our names to smaller buffers to reduce our memory footprint.
 540          */
 541         if (inst->ri_common_name != NULL) {
 542                 char *tmp = safe_strdup(inst->ri_common_name);
 543                 startd_free(inst->ri_common_name, max_scf_value_size);
 544                 inst->ri_common_name = tmp;
 545         }
 546 
 547         if (inst->ri_C_common_name != NULL) {
 548                 char *tmp = safe_strdup(inst->ri_C_common_name);
 549                 startd_free(inst->ri_C_common_name, max_scf_value_size);
 550                 inst->ri_C_common_name = tmp;
 551         }
 552 
 553         switch (r) {
 554         case 0:
 555                 break;
 556 
 557         case ECONNABORTED:
 558                 libscf_handle_rebind(h);
 559                 goto rep_retry;
 560 
 561         case ECANCELED:
 562                 goto deleted;
 563 
 564         case ECHILD:
 565         case ENOENT:
 566                 break;
 567 
 568         default:
 569                 assert(0);
 570                 abort();
 571         }
 572 
 573         switch (libscf_read_method_ids(h, scf_inst, inst->ri_i.i_fmri,
 574             &inst->ri_i.i_primary_ctid, &inst->ri_i.i_transient_ctid,
 575             &start_pid)) {
 576         case 0:
 577                 break;
 578 
 579         case ECONNABORTED:
 580                 libscf_handle_rebind(h);
 581                 goto rep_retry;
 582 
 583         case ECANCELED:
 584                 goto deleted;
 585 
 586         default:
 587                 assert(0);
 588                 abort();
 589         }
 590 
 591         if (inst->ri_i.i_primary_ctid >= 1) {
 592                 contract_hash_store(inst->ri_i.i_primary_ctid, inst->ri_id);
 593 
 594                 switch (check_contract(inst, B_TRUE, scf_inst)) {
 595                 case 0:
 596                         break;
 597 
 598                 case ECONNABORTED:
 599                         libscf_handle_rebind(h);
 600                         goto rep_retry;
 601 
 602                 case ECANCELED:
 603                         goto deleted;
 604 
 605                 default:
 606                         assert(0);
 607                         abort();
 608                 }
 609         }
 610 
 611         if (inst->ri_i.i_transient_ctid >= 1) {
 612                 switch (check_contract(inst, B_FALSE, scf_inst)) {
 613                 case 0:
 614                         break;
 615 
 616                 case ECONNABORTED:
 617                         libscf_handle_rebind(h);
 618                         goto rep_retry;
 619 
 620                 case ECANCELED:
 621                         goto deleted;
 622 
 623                 default:
 624                         assert(0);
 625                         abort();
 626                 }
 627         }
 628 
 629         /* No more failures we live through, so add it to the list. */
 630         (void) pthread_mutex_init(&inst->ri_lock, &mutex_attrs);
 631         (void) pthread_mutex_init(&inst->ri_queue_lock, &mutex_attrs);
 632         MUTEX_LOCK(&inst->ri_lock);
 633         MUTEX_LOCK(&inst->ri_queue_lock);
 634 
 635         (void) pthread_cond_init(&inst->ri_method_cv, NULL);
 636 
 637         uu_list_node_init(inst, &inst->ri_link, restarter_instance_pool);
 638         uu_list_insert(instance_list.ril_instance_list, inst, idx);
 639         MUTEX_UNLOCK(&instance_list.ril_lock);
 640 
 641         if (start_pid != -1 &&
 642             (inst->ri_flags & RINST_STYLE_MASK) == RINST_WAIT) {
 643                 int ret;
 644                 ret = wait_register(start_pid, inst->ri_i.i_fmri, 0, 1);
 645                 if (ret == -1) {
 646                         /*
 647                          * Implication:  if we can't reregister the
 648                          * instance, we will start another one.  Two
 649                          * instances may or may not result in a resource
 650                          * conflict.
 651                          */
 652                         log_error(LOG_WARNING,
 653                             "%s: couldn't reregister %ld for wait\n",
 654                             inst->ri_i.i_fmri, start_pid);
 655                 } else if (ret == 1) {
 656                         /*
 657                          * Leading PID has exited.
 658                          */
 659                         (void) stop_instance(h, inst, RSTOP_EXIT);
 660                 }
 661         }
 662 
 663 
 664         scf_pg_destroy(pg);
 665 
 666         if (do_commit_states)
 667                 (void) restarter_instance_update_states(h, inst, state,
 668                     next_state, RERR_NONE, reason);
 669 
 670         log_framework(LOG_DEBUG, "%s is a %s-style service\n", name,
 671             service_style(inst->ri_flags));
 672 
 673         MUTEX_UNLOCK(&inst->ri_queue_lock);
 674         MUTEX_UNLOCK(&inst->ri_lock);
 675 
 676         startd_free(svc_name, max_scf_name_size);
 677         startd_free(inst_name, max_scf_name_size);
 678         scf_snapshot_destroy(snap);
 679         scf_instance_destroy(scf_inst);
 680         scf_service_destroy(scf_svc);
 681 
 682         log_framework(LOG_DEBUG, "%s: inserted instance into restarter list\n",
 683             name);
 684 
 685         return (0);
 686 
 687 deleted:
 688         MUTEX_UNLOCK(&instance_list.ril_lock);
 689         startd_free(inst_name, max_scf_name_size);
 690         startd_free(svc_name, max_scf_name_size);
 691         if (snap != NULL)
 692                 scf_snapshot_destroy(snap);
 693         scf_pg_destroy(pg);
 694         scf_instance_destroy(scf_inst);
 695         scf_service_destroy(scf_svc);
 696         startd_free((void *)inst->ri_i.i_fmri, strlen(inst->ri_i.i_fmri) + 1);
 697         uu_list_destroy(inst->ri_queue);
 698         if (inst->ri_logstem != NULL)
 699                 startd_free(inst->ri_logstem, PATH_MAX);
 700         if (inst->ri_common_name != NULL)
 701                 free(inst->ri_common_name);
 702         if (inst->ri_C_common_name != NULL)
 703                 free(inst->ri_C_common_name);
 704         startd_free(inst->ri_utmpx_prefix, max_scf_value_size);
 705         startd_free(inst, sizeof (restarter_inst_t));
 706         return (ENOENT);
 707 }
 708 
 709 static void
 710 restarter_delete_inst(restarter_inst_t *ri)
 711 {
 712         int id;
 713         restarter_inst_t *rip;
 714         void *cookie = NULL;
 715         restarter_instance_qentry_t *e;
 716 
 717         assert(MUTEX_HELD(&ri->ri_lock));
 718 
 719         /*
 720          * Must drop the instance lock so we can pick up the instance_list
 721          * lock & remove the instance.
 722          */
 723         id = ri->ri_id;
 724         MUTEX_UNLOCK(&ri->ri_lock);
 725 
 726         MUTEX_LOCK(&instance_list.ril_lock);
 727 
 728         rip = uu_list_find(instance_list.ril_instance_list, &id, NULL, NULL);
 729         if (rip == NULL) {
 730                 MUTEX_UNLOCK(&instance_list.ril_lock);
 731                 return;
 732         }
 733 
 734         assert(ri == rip);
 735 
 736         uu_list_remove(instance_list.ril_instance_list, ri);
 737 
 738         log_framework(LOG_DEBUG, "%s: deleted instance from restarter list\n",
 739             ri->ri_i.i_fmri);
 740 
 741         MUTEX_UNLOCK(&instance_list.ril_lock);
 742 
 743         /*
 744          * We can lock the instance without holding the instance_list lock
 745          * since we removed the instance from the list.
 746          */
 747         MUTEX_LOCK(&ri->ri_lock);
 748         MUTEX_LOCK(&ri->ri_queue_lock);
 749 
 750         if (ri->ri_i.i_primary_ctid >= 1)
 751                 contract_hash_remove(ri->ri_i.i_primary_ctid);
 752 
 753         while (ri->ri_method_thread != 0 || ri->ri_method_waiters > 0)
 754                 (void) pthread_cond_wait(&ri->ri_method_cv, &ri->ri_lock);
 755 
 756         while ((e = uu_list_teardown(ri->ri_queue, &cookie)) != NULL)
 757                 startd_free(e, sizeof (*e));
 758         uu_list_destroy(ri->ri_queue);
 759 
 760         startd_free((void *)ri->ri_i.i_fmri, strlen(ri->ri_i.i_fmri) + 1);
 761         startd_free(ri->ri_logstem, PATH_MAX);
 762         if (ri->ri_common_name != NULL)
 763                 free(ri->ri_common_name);
 764         if (ri->ri_C_common_name != NULL)
 765                 free(ri->ri_C_common_name);
 766         startd_free(ri->ri_utmpx_prefix, max_scf_value_size);
 767         (void) pthread_mutex_destroy(&ri->ri_lock);
 768         (void) pthread_mutex_destroy(&ri->ri_queue_lock);
 769         startd_free(ri, sizeof (restarter_inst_t));
 770 }
 771 
 772 /*
 773  * instance_is_wait_style()
 774  *
 775  *   Returns 1 if the given instance is a "wait-style" service instance.
 776  */
 777 int
 778 instance_is_wait_style(restarter_inst_t *inst)
 779 {
 780         assert(MUTEX_HELD(&inst->ri_lock));
 781         return ((inst->ri_flags & RINST_STYLE_MASK) == RINST_WAIT);
 782 }
 783 
 784 /*
 785  * instance_is_transient_style()
 786  *
 787  *   Returns 1 if the given instance is a transient service instance.
 788  */
 789 int
 790 instance_is_transient_style(restarter_inst_t *inst)
 791 {
 792         assert(MUTEX_HELD(&inst->ri_lock));
 793         return ((inst->ri_flags & RINST_STYLE_MASK) == RINST_TRANSIENT);
 794 }
 795 
 796 /*
 797  * instance_in_transition()
 798  * Returns 1 if instance is in transition, 0 if not
 799  */
 800 int
 801 instance_in_transition(restarter_inst_t *inst)
 802 {
 803         assert(MUTEX_HELD(&inst->ri_lock));
 804         if (inst->ri_i.i_next_state == RESTARTER_STATE_NONE)
 805                 return (0);
 806         return (1);
 807 }
 808 
 809 /*
 810  * returns 1 if instance is already started, 0 if not
 811  */
 812 static int
 813 instance_started(restarter_inst_t *inst)
 814 {
 815         int ret;
 816 
 817         assert(MUTEX_HELD(&inst->ri_lock));
 818 
 819         if (inst->ri_i.i_state == RESTARTER_STATE_ONLINE ||
 820             inst->ri_i.i_state == RESTARTER_STATE_DEGRADED)
 821                 ret = 1;
 822         else
 823                 ret = 0;
 824 
 825         return (ret);
 826 }
 827 
 828 /*
 829  * Returns
 830  *   0 - success
 831  *   ECONNRESET - success, but h was rebound
 832  */
 833 int
 834 restarter_instance_update_states(scf_handle_t *h, restarter_inst_t *ri,
 835     restarter_instance_state_t new_state,
 836     restarter_instance_state_t new_state_next, restarter_error_t err,
 837     restarter_str_t reason)
 838 {
 839         protocol_states_t *states;
 840         int e;
 841         uint_t retry_count = 0, msecs = ALLOC_DELAY;
 842         boolean_t rebound = B_FALSE;
 843         int prev_state_online;
 844         int state_online;
 845 
 846         assert(MUTEX_HELD(&ri->ri_lock));
 847 
 848         prev_state_online = instance_started(ri);
 849 
 850 retry:
 851         e = _restarter_commit_states(h, &ri->ri_i, new_state, new_state_next,
 852             restarter_get_str_short(reason));
 853         switch (e) {
 854         case 0:
 855                 break;
 856 
 857         case ENOMEM:
 858                 ++retry_count;
 859                 if (retry_count < ALLOC_RETRY) {
 860                         (void) poll(NULL, 0, msecs);
 861                         msecs *= ALLOC_DELAY_MULT;
 862                         goto retry;
 863                 }
 864 
 865                 /* Like startd_alloc(). */
 866                 uu_die("Insufficient memory.\n");
 867                 /* NOTREACHED */
 868 
 869         case ECONNABORTED:
 870                 libscf_handle_rebind(h);
 871                 rebound = B_TRUE;
 872                 goto retry;
 873 
 874         case EPERM:
 875         case EACCES:
 876         case EROFS:
 877                 log_error(LOG_NOTICE, "Could not commit state change for %s "
 878                     "to repository: %s.\n", ri->ri_i.i_fmri, strerror(e));
 879                 /* FALLTHROUGH */
 880 
 881         case ENOENT:
 882                 ri->ri_i.i_state = new_state;
 883                 ri->ri_i.i_next_state = new_state_next;
 884                 break;
 885 
 886         case EINVAL:
 887         default:
 888                 bad_error("_restarter_commit_states", e);
 889         }
 890 
 891         states = startd_alloc(sizeof (protocol_states_t));
 892         states->ps_state = new_state;
 893         states->ps_state_next = new_state_next;
 894         states->ps_err = err;
 895         states->ps_reason = reason;
 896         graph_protocol_send_event(ri->ri_i.i_fmri, GRAPH_UPDATE_STATE_CHANGE,
 897             (void *)states);
 898 
 899         state_online = instance_started(ri);
 900 
 901         if (prev_state_online && !state_online)
 902                 ri->ri_post_offline_hook();
 903         else if (!prev_state_online && state_online)
 904                 ri->ri_post_online_hook();
 905 
 906         return (rebound ? ECONNRESET : 0);
 907 }
 908 
 909 void
 910 restarter_mark_pending_snapshot(const char *fmri, uint_t flag)
 911 {
 912         restarter_inst_t *inst;
 913 
 914         assert(flag == RINST_RETAKE_RUNNING || flag == RINST_RETAKE_START);
 915 
 916         inst = inst_lookup_by_name(fmri);
 917         if (inst == NULL)
 918                 return;
 919 
 920         inst->ri_flags |= flag;
 921 
 922         MUTEX_UNLOCK(&inst->ri_lock);
 923 }
 924 
 925 static void
 926 restarter_take_pending_snapshots(scf_handle_t *h)
 927 {
 928         restarter_inst_t *inst;
 929         int r;
 930 
 931         MUTEX_LOCK(&instance_list.ril_lock);
 932 
 933         for (inst = uu_list_first(instance_list.ril_instance_list);
 934             inst != NULL;
 935             inst = uu_list_next(instance_list.ril_instance_list, inst)) {
 936                 const char *fmri;
 937                 scf_instance_t *sinst = NULL;
 938 
 939                 MUTEX_LOCK(&inst->ri_lock);
 940 
 941                 /*
 942                  * This is where we'd check inst->ri_method_thread and if it
 943                  * were nonzero we'd wait in anticipation of another thread
 944                  * executing a method for inst.  Doing so with the instance_list
 945                  * locked, though, leads to deadlock.  Since taking a snapshot
 946                  * during that window won't hurt anything, we'll just continue.
 947                  */
 948 
 949                 fmri = inst->ri_i.i_fmri;
 950 
 951                 if (inst->ri_flags & RINST_RETAKE_RUNNING) {
 952                         scf_snapshot_t *rsnap;
 953 
 954                         (void) libscf_fmri_get_instance(h, fmri, &sinst);
 955 
 956                         rsnap = libscf_get_or_make_running_snapshot(sinst,
 957                             fmri, B_FALSE);
 958 
 959                         scf_instance_destroy(sinst);
 960 
 961                         if (rsnap != NULL)
 962                                 inst->ri_flags &= ~RINST_RETAKE_RUNNING;
 963 
 964                         scf_snapshot_destroy(rsnap);
 965                 }
 966 
 967                 if (inst->ri_flags & RINST_RETAKE_START) {
 968                         switch (r = libscf_snapshots_poststart(h, fmri,
 969                             B_FALSE)) {
 970                         case 0:
 971                         case ENOENT:
 972                                 inst->ri_flags &= ~RINST_RETAKE_START;
 973                                 break;
 974 
 975                         case ECONNABORTED:
 976                                 break;
 977 
 978                         case EACCES:
 979                         default:
 980                                 bad_error("libscf_snapshots_poststart", r);
 981                         }
 982                 }
 983 
 984                 MUTEX_UNLOCK(&inst->ri_lock);
 985         }
 986 
 987         MUTEX_UNLOCK(&instance_list.ril_lock);
 988 }
 989 
 990 /* ARGSUSED */
 991 void *
 992 restarter_post_fsminimal_thread(void *unused)
 993 {
 994         scf_handle_t *h;
 995         int r;
 996 
 997         (void) pthread_setname_np(pthread_self(), "restarter_post_fsmin");
 998 
 999         h = libscf_handle_create_bound_loop();
1000 
1001         for (;;) {
1002                 r = libscf_create_self(h);
1003                 if (r == 0)
1004                         break;
1005 
1006                 assert(r == ECONNABORTED);
1007                 libscf_handle_rebind(h);
1008         }
1009 
1010         restarter_take_pending_snapshots(h);
1011 
1012         (void) scf_handle_unbind(h);
1013         scf_handle_destroy(h);
1014 
1015         return (NULL);
1016 }
1017 
1018 /*
1019  * int stop_instance()
1020  *
1021  *   Stop the instance identified by the instance given as the second argument,
1022  *   for the cause stated.
1023  *
1024  *   Returns
1025  *     0 - success
1026  *     -1 - inst is in transition
1027  */
1028 static int
1029 stop_instance(scf_handle_t *local_handle, restarter_inst_t *inst,
1030     stop_cause_t cause)
1031 {
1032         fork_info_t *info;
1033         const char *cp;
1034         int err;
1035         restarter_error_t re;
1036         restarter_str_t reason;
1037         restarter_instance_state_t new_state;
1038 
1039         assert(MUTEX_HELD(&inst->ri_lock));
1040         assert(inst->ri_method_thread == 0);
1041 
1042         switch (cause) {
1043         case RSTOP_EXIT:
1044                 re = RERR_RESTART;
1045                 reason = restarter_str_ct_ev_exit;
1046                 cp = "all processes in service exited";
1047                 break;
1048         case RSTOP_ERR_CFG:
1049                 re = RERR_FAULT;
1050                 reason = restarter_str_method_failed;
1051                 cp = "service exited with a configuration error";
1052                 break;
1053         case RSTOP_ERR_EXIT:
1054                 re = RERR_RESTART;
1055                 reason = restarter_str_ct_ev_exit;
1056                 cp = "service exited with an error";
1057                 break;
1058         case RSTOP_CORE:
1059                 re = RERR_FAULT;
1060                 reason = restarter_str_ct_ev_core;
1061                 cp = "process dumped core";
1062                 break;
1063         case RSTOP_SIGNAL:
1064                 re = RERR_FAULT;
1065                 reason = restarter_str_ct_ev_signal;
1066                 cp = "process received fatal signal from outside the service";
1067                 break;
1068         case RSTOP_HWERR:
1069                 re = RERR_FAULT;
1070                 reason = restarter_str_ct_ev_hwerr;
1071                 cp = "process killed due to uncorrectable hardware error";
1072                 break;
1073         case RSTOP_DEPENDENCY:
1074                 re = RERR_RESTART;
1075                 reason = restarter_str_dependency_activity;
1076                 cp = "dependency activity requires stop";
1077                 break;
1078         case RSTOP_DISABLE:
1079                 re = RERR_RESTART;
1080                 reason = restarter_str_disable_request;
1081                 cp = "service disabled";
1082                 break;
1083         case RSTOP_RESTART:
1084                 re = RERR_RESTART;
1085                 reason = restarter_str_restart_request;
1086                 cp = "service restarting";
1087                 break;
1088         default:
1089 #ifndef NDEBUG
1090                 (void) fprintf(stderr, "Unknown cause %d at %s:%d.\n",
1091                     cause, __FILE__, __LINE__);
1092 #endif
1093                 abort();
1094         }
1095 
1096         /* Services in the disabled and maintenance state are ignored */
1097         if (inst->ri_i.i_state == RESTARTER_STATE_MAINT ||
1098             inst->ri_i.i_state == RESTARTER_STATE_DISABLED) {
1099                 log_framework(LOG_DEBUG,
1100                     "%s: stop_instance -> is maint/disabled\n",
1101                     inst->ri_i.i_fmri);
1102                 return (0);
1103         }
1104 
1105         /* Already stopped instances are left alone */
1106         if (instance_started(inst) == 0) {
1107                 log_framework(LOG_DEBUG, "Restarter: %s is already stopped.\n",
1108                     inst->ri_i.i_fmri);
1109                 return (0);
1110         }
1111 
1112         if (instance_in_transition(inst)) {
1113                 /* requeue event by returning -1 */
1114                 log_framework(LOG_DEBUG,
1115                     "Restarter: Not stopping %s, in transition.\n",
1116                     inst->ri_i.i_fmri);
1117                 return (-1);
1118         }
1119 
1120         log_instance(inst, B_TRUE, "Stopping because %s.", cp);
1121 
1122         log_framework(re == RERR_FAULT ? LOG_INFO : LOG_DEBUG,
1123             "%s: Instance stopping because %s.\n", inst->ri_i.i_fmri, cp);
1124 
1125         if (instance_is_wait_style(inst) &&
1126             (cause == RSTOP_EXIT ||
1127             cause == RSTOP_ERR_CFG ||
1128             cause == RSTOP_ERR_EXIT)) {
1129                 /*
1130                  * No need to stop instance, as child has exited; remove
1131                  * contract and move the instance to the offline state.
1132                  */
1133                 switch (err = restarter_instance_update_states(local_handle,
1134                     inst, inst->ri_i.i_state, RESTARTER_STATE_OFFLINE, re,
1135                     reason)) {
1136                 case 0:
1137                 case ECONNRESET:
1138                         break;
1139 
1140                 default:
1141                         bad_error("restarter_instance_update_states", err);
1142                 }
1143 
1144                 if (cause == RSTOP_ERR_EXIT) {
1145                         /*
1146                          * The RSTOP_ERR_EXIT cause is set via the
1147                          * wait_thread -> wait_remove code path when we have
1148                          * a "wait" style svc that exited with an error. If
1149                          * the svc is failing too quickly, we throttle it so
1150                          * that we don't restart it more than once/second.
1151                          * Since we know we're running in the wait thread its
1152                          * ok to throttle it right here.
1153                          */
1154                         (void) update_fault_count(inst, FAULT_COUNT_INCR);
1155                         if (method_rate_critical(inst)) {
1156                                 log_instance(inst, B_TRUE, "Failing too "
1157                                     "quickly, throttling.");
1158                                 (void) sleep(WT_SVC_ERR_THROTTLE);
1159                         }
1160                 } else {
1161                         (void) update_fault_count(inst, FAULT_COUNT_RESET);
1162                         reset_start_times(inst);
1163                 }
1164 
1165                 if (inst->ri_i.i_primary_ctid != 0) {
1166                         inst->ri_m_inst =
1167                             safe_scf_instance_create(local_handle);
1168                         inst->ri_mi_deleted = B_FALSE;
1169 
1170                         libscf_reget_instance(inst);
1171                         method_remove_contract(inst, B_TRUE, B_TRUE);
1172 
1173                         scf_instance_destroy(inst->ri_m_inst);
1174                         inst->ri_m_inst = NULL;
1175                 }
1176 
1177                 switch (err = restarter_instance_update_states(local_handle,
1178                     inst, inst->ri_i.i_next_state, RESTARTER_STATE_NONE, re,
1179                     reason)) {
1180                 case 0:
1181                 case ECONNRESET:
1182                         break;
1183 
1184                 default:
1185                         bad_error("restarter_instance_update_states", err);
1186                 }
1187 
1188                 if (cause != RSTOP_ERR_CFG)
1189                         return (0);
1190         } else if (instance_is_wait_style(inst) && re == RERR_RESTART) {
1191                 /*
1192                  * Stopping a wait service through means other than the pid
1193                  * exiting should keep wait_thread() from restarting the
1194                  * service, by removing it from the wait list.
1195                  * We cannot remove it right now otherwise the process will
1196                  * end up <defunct> so mark it to be ignored.
1197                  */
1198                 wait_ignore_by_fmri(inst->ri_i.i_fmri);
1199         }
1200 
1201         /*
1202          * There are some configuration errors which we cannot detect until we
1203          * try to run the method.  For example, see exec_method() where the
1204          * restarter_set_method_context() call can return SMF_EXIT_ERR_CONFIG
1205          * in several cases. If this happens for a "wait-style" svc,
1206          * wait_remove() sets the cause as RSTOP_ERR_CFG so that we can detect
1207          * the configuration error and go into maintenance, even though it is
1208          * a "wait-style" svc.
1209          */
1210         if (cause == RSTOP_ERR_CFG)
1211                 new_state = RESTARTER_STATE_MAINT;
1212         else
1213                 new_state = inst->ri_i.i_enabled ?
1214                     RESTARTER_STATE_OFFLINE : RESTARTER_STATE_DISABLED;
1215 
1216         switch (err = restarter_instance_update_states(local_handle, inst,
1217             inst->ri_i.i_state, new_state, RERR_NONE, reason)) {
1218         case 0:
1219         case ECONNRESET:
1220                 break;
1221 
1222         default:
1223                 bad_error("restarter_instance_update_states", err);
1224         }
1225 
1226         info = startd_zalloc(sizeof (fork_info_t));
1227 
1228         info->sf_id = inst->ri_id;
1229         info->sf_method_type = METHOD_STOP;
1230         info->sf_event_type = re;
1231         info->sf_reason = reason;
1232         inst->ri_method_thread = startd_thread_create(method_thread, info);
1233 
1234         return (0);
1235 }
1236 
1237 /*
1238  * Returns
1239  *   ENOENT - fmri is not in instance_list
1240  *   0 - success
1241  *   ECONNRESET - success, though handle was rebound
1242  *   -1 - instance is in transition
1243  */
1244 int
1245 stop_instance_fmri(scf_handle_t *h, const char *fmri, uint_t flags)
1246 {
1247         restarter_inst_t *rip;
1248         int r;
1249 
1250         rip = inst_lookup_by_name(fmri);
1251         if (rip == NULL)
1252                 return (ENOENT);
1253 
1254         r = stop_instance(h, rip, flags);
1255 
1256         MUTEX_UNLOCK(&rip->ri_lock);
1257 
1258         return (r);
1259 }
1260 
1261 static void
1262 unmaintain_instance(scf_handle_t *h, restarter_inst_t *rip,
1263     unmaint_cause_t cause)
1264 {
1265         ctid_t ctid;
1266         scf_instance_t *inst;
1267         int r;
1268         uint_t tries = 0, msecs = ALLOC_DELAY;
1269         const char *cp;
1270         restarter_str_t reason;
1271 
1272         assert(MUTEX_HELD(&rip->ri_lock));
1273 
1274         if (rip->ri_i.i_state != RESTARTER_STATE_MAINT) {
1275                 log_error(LOG_DEBUG, "Restarter: "
1276                     "Ignoring maintenance off command because %s is not in the "
1277                     "maintenance state.\n", rip->ri_i.i_fmri);
1278                 return;
1279         }
1280 
1281         switch (cause) {
1282         case RUNMAINT_CLEAR:
1283                 cp = "clear requested";
1284                 reason = restarter_str_clear_request;
1285                 break;
1286         case RUNMAINT_DISABLE:
1287                 cp = "disable requested";
1288                 reason = restarter_str_disable_request;
1289                 break;
1290         default:
1291 #ifndef NDEBUG
1292                 (void) fprintf(stderr, "Uncaught case for %d at %s:%d.\n",
1293                     cause, __FILE__, __LINE__);
1294 #endif
1295                 abort();
1296         }
1297 
1298         log_instance(rip, B_TRUE, "Leaving maintenance because %s.",
1299             cp);
1300         log_framework(LOG_DEBUG, "%s: Instance leaving maintenance because "
1301             "%s.\n", rip->ri_i.i_fmri, cp);
1302 
1303         (void) restarter_instance_update_states(h, rip, RESTARTER_STATE_UNINIT,
1304             RESTARTER_STATE_NONE, RERR_RESTART, reason);
1305 
1306         /*
1307          * If we did ADMIN_MAINT_ON_IMMEDIATE, then there might still be
1308          * a primary contract.
1309          */
1310         if (rip->ri_i.i_primary_ctid == 0)
1311                 return;
1312 
1313         ctid = rip->ri_i.i_primary_ctid;
1314         contract_abandon(ctid);
1315         rip->ri_i.i_primary_ctid = 0;
1316 
1317 rep_retry:
1318         switch (r = libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst)) {
1319         case 0:
1320                 break;
1321 
1322         case ECONNABORTED:
1323                 libscf_handle_rebind(h);
1324                 goto rep_retry;
1325 
1326         case ENOENT:
1327                 /* Must have been deleted. */
1328                 return;
1329 
1330         case EINVAL:
1331         case ENOTSUP:
1332         default:
1333                 bad_error("libscf_handle_rebind", r);
1334         }
1335 
1336 again:
1337         r = restarter_remove_contract(inst, ctid, RESTARTER_CONTRACT_PRIMARY);
1338         switch (r) {
1339         case 0:
1340                 break;
1341 
1342         case ENOMEM:
1343                 ++tries;
1344                 if (tries < ALLOC_RETRY) {
1345                         (void) poll(NULL, 0, msecs);
1346                         msecs *= ALLOC_DELAY_MULT;
1347                         goto again;
1348                 }
1349 
1350                 uu_die("Insufficient memory.\n");
1351                 /* NOTREACHED */
1352 
1353         case ECONNABORTED:
1354                 scf_instance_destroy(inst);
1355                 libscf_handle_rebind(h);
1356                 goto rep_retry;
1357 
1358         case ECANCELED:
1359                 break;
1360 
1361         case EPERM:
1362         case EACCES:
1363         case EROFS:
1364                 log_error(LOG_INFO,
1365                     "Could not remove contract id %lu for %s (%s).\n", ctid,
1366                     rip->ri_i.i_fmri, strerror(r));
1367                 break;
1368 
1369         case EINVAL:
1370         case EBADF:
1371         default:
1372                 bad_error("restarter_remove_contract", r);
1373         }
1374 
1375         scf_instance_destroy(inst);
1376 }
1377 
1378 /*
1379  * enable_inst()
1380  *   Set inst->ri_i.i_enabled.  Expects 'e' to be _ENABLE, _DISABLE, or
1381  *   _ADMIN_DISABLE.  If the event is _ENABLE and inst is uninitialized or
1382  *   disabled, move it to offline.  If the event is _DISABLE or
1383  *   _ADMIN_DISABLE, make sure inst will move to disabled.
1384  *
1385  *   Returns
1386  *     0 - success
1387  *     ECONNRESET - h was rebound
1388  */
1389 static int
1390 enable_inst(scf_handle_t *h, restarter_inst_t *inst,
1391     restarter_instance_qentry_t *riq)
1392 {
1393         restarter_instance_state_t state;
1394         restarter_event_type_t e = riq->riq_type;
1395         restarter_str_t reason = restarter_str_per_configuration;
1396         int r;
1397 
1398         assert(MUTEX_HELD(&inst->ri_lock));
1399         assert(e == RESTARTER_EVENT_TYPE_ADMIN_DISABLE ||
1400             e == RESTARTER_EVENT_TYPE_DISABLE ||
1401             e == RESTARTER_EVENT_TYPE_ENABLE);
1402         assert(instance_in_transition(inst) == 0);
1403 
1404         state = inst->ri_i.i_state;
1405 
1406         if (e == RESTARTER_EVENT_TYPE_ENABLE) {
1407                 inst->ri_i.i_enabled = 1;
1408 
1409                 if (state == RESTARTER_STATE_UNINIT ||
1410                     state == RESTARTER_STATE_DISABLED) {
1411                         /*
1412                          * B_FALSE: Don't log an error if the log_instance()
1413                          * fails because it will fail on the miniroot before
1414                          * install-discovery runs.
1415                          */
1416                         log_instance(inst, B_FALSE, "Enabled.");
1417                         log_framework(LOG_DEBUG, "%s: Instance enabled.\n",
1418                             inst->ri_i.i_fmri);
1419 
1420                         /*
1421                          * If we are coming from DISABLED, it was obviously an
1422                          * enable request. If we are coming from UNINIT, it may
1423                          * have been a sevice in MAINT that was cleared.
1424                          */
1425                         if (riq->riq_reason == restarter_str_clear_request)
1426                                 reason = restarter_str_clear_request;
1427                         else if (state == RESTARTER_STATE_DISABLED)
1428                                 reason = restarter_str_enable_request;
1429                         (void) restarter_instance_update_states(h, inst,
1430                             RESTARTER_STATE_OFFLINE, RESTARTER_STATE_NONE,
1431                             RERR_NONE, reason);
1432                 } else {
1433                         log_framework(LOG_DEBUG, "Restarter: "
1434                             "Not changing state of %s for enable command.\n",
1435                             inst->ri_i.i_fmri);
1436                 }
1437         } else {
1438                 inst->ri_i.i_enabled = 0;
1439 
1440                 switch (state) {
1441                 case RESTARTER_STATE_ONLINE:
1442                 case RESTARTER_STATE_DEGRADED:
1443                         r = stop_instance(h, inst, RSTOP_DISABLE);
1444                         return (r == ECONNRESET ? 0 : r);
1445 
1446                 case RESTARTER_STATE_OFFLINE:
1447                 case RESTARTER_STATE_UNINIT:
1448                         if (inst->ri_i.i_primary_ctid != 0) {
1449                                 inst->ri_m_inst = safe_scf_instance_create(h);
1450                                 inst->ri_mi_deleted = B_FALSE;
1451 
1452                                 libscf_reget_instance(inst);
1453                                 method_remove_contract(inst, B_TRUE, B_TRUE);
1454 
1455                                 scf_instance_destroy(inst->ri_m_inst);
1456                         }
1457                         /* B_FALSE: See log_instance(..., "Enabled."); above */
1458                         log_instance(inst, B_FALSE, "Disabled.");
1459                         log_framework(LOG_DEBUG, "%s: Instance disabled.\n",
1460                             inst->ri_i.i_fmri);
1461 
1462                         /*
1463                          * If we are coming from OFFLINE, it was obviously a
1464                          * disable request. But if we are coming from
1465                          * UNINIT, it may have been a disable request for a
1466                          * service in MAINT.
1467                          */
1468                         if (riq->riq_reason == restarter_str_disable_request ||
1469                             state == RESTARTER_STATE_OFFLINE)
1470                                 reason = restarter_str_disable_request;
1471                         (void) restarter_instance_update_states(h, inst,
1472                             RESTARTER_STATE_DISABLED, RESTARTER_STATE_NONE,
1473                             RERR_RESTART, reason);
1474                         return (0);
1475 
1476                 case RESTARTER_STATE_DISABLED:
1477                         break;
1478 
1479                 case RESTARTER_STATE_MAINT:
1480                         /*
1481                          * We only want to pull the instance out of maintenance
1482                          * if the disable is on adminstrative request.  The
1483                          * graph engine sends _DISABLE events whenever a
1484                          * service isn't in the disabled state, and we don't
1485                          * want to pull the service out of maintenance if,
1486                          * for example, it is there due to a dependency cycle.
1487                          */
1488                         if (e == RESTARTER_EVENT_TYPE_ADMIN_DISABLE)
1489                                 unmaintain_instance(h, inst, RUNMAINT_DISABLE);
1490                         break;
1491 
1492                 default:
1493 #ifndef NDEBUG
1494                         (void) fprintf(stderr, "Restarter instance %s has "
1495                             "unknown state %d.\n", inst->ri_i.i_fmri, state);
1496 #endif
1497                         abort();
1498                 }
1499         }
1500 
1501         return (0);
1502 }
1503 
1504 static void
1505 start_instance(scf_handle_t *local_handle, restarter_inst_t *inst,
1506     int32_t reason)
1507 {
1508         fork_info_t *info;
1509         restarter_str_t new_reason;
1510 
1511         assert(MUTEX_HELD(&inst->ri_lock));
1512         assert(instance_in_transition(inst) == 0);
1513         assert(inst->ri_method_thread == 0);
1514 
1515         log_framework(LOG_DEBUG, "%s: trying to start instance\n",
1516             inst->ri_i.i_fmri);
1517 
1518         /*
1519          * We want to keep the original reason for restarts and clear actions
1520          */
1521         switch (reason) {
1522         case restarter_str_restart_request:
1523         case restarter_str_clear_request:
1524                 new_reason = reason;
1525                 break;
1526         default:
1527                 new_reason = restarter_str_dependencies_satisfied;
1528         }
1529 
1530         /* Services in the disabled and maintenance state are ignored */
1531         if (inst->ri_i.i_state == RESTARTER_STATE_MAINT ||
1532             inst->ri_i.i_state == RESTARTER_STATE_DISABLED ||
1533             inst->ri_i.i_enabled == 0) {
1534                 log_framework(LOG_DEBUG,
1535                     "%s: start_instance -> is maint/disabled\n",
1536                     inst->ri_i.i_fmri);
1537                 return;
1538         }
1539 
1540         /* Already started instances are left alone */
1541         if (instance_started(inst) == 1) {
1542                 log_framework(LOG_DEBUG,
1543                     "%s: start_instance -> is already started\n",
1544                     inst->ri_i.i_fmri);
1545                 return;
1546         }
1547 
1548         log_framework(LOG_DEBUG, "%s: starting instance.\n", inst->ri_i.i_fmri);
1549 
1550         (void) restarter_instance_update_states(local_handle, inst,
1551             inst->ri_i.i_state, RESTARTER_STATE_ONLINE, RERR_NONE, new_reason);
1552 
1553         info = startd_zalloc(sizeof (fork_info_t));
1554 
1555         info->sf_id = inst->ri_id;
1556         info->sf_method_type = METHOD_START;
1557         info->sf_event_type = RERR_NONE;
1558         info->sf_reason = new_reason;
1559         inst->ri_method_thread = startd_thread_create(method_thread, info);
1560 }
1561 
1562 static int
1563 event_from_tty(scf_handle_t *h, restarter_inst_t *rip)
1564 {
1565         scf_instance_t *inst;
1566         int ret = 0;
1567 
1568         if (libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst))
1569                 return (-1);
1570 
1571         ret = restarter_inst_ractions_from_tty(inst);
1572 
1573         scf_instance_destroy(inst);
1574         return (ret);
1575 }
1576 
1577 static boolean_t
1578 restart_dump(scf_handle_t *h, restarter_inst_t *rip)
1579 {
1580         scf_instance_t *inst;
1581         boolean_t ret = B_FALSE;
1582 
1583         if (libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst))
1584                 return (-1);
1585 
1586         if (restarter_inst_dump(inst) == 1)
1587                 ret = B_TRUE;
1588 
1589         scf_instance_destroy(inst);
1590         return (ret);
1591 }
1592 
1593 static void
1594 maintain_instance(scf_handle_t *h, restarter_inst_t *rip, int immediate,
1595     restarter_str_t reason)
1596 {
1597         fork_info_t *info;
1598         scf_instance_t *scf_inst = NULL;
1599 
1600         assert(MUTEX_HELD(&rip->ri_lock));
1601         assert(reason != restarter_str_none);
1602         assert(rip->ri_method_thread == 0);
1603 
1604         log_instance(rip, B_TRUE, "Stopping for maintenance due to %s.",
1605             restarter_get_str_short(reason));
1606         log_framework(LOG_DEBUG, "%s: stopping for maintenance due to %s.\n",
1607             rip->ri_i.i_fmri, restarter_get_str_short(reason));
1608 
1609         /* Services in the maintenance state are ignored */
1610         if (rip->ri_i.i_state == RESTARTER_STATE_MAINT) {
1611                 log_framework(LOG_DEBUG,
1612                     "%s: maintain_instance -> is already in maintenance\n",
1613                     rip->ri_i.i_fmri);
1614                 return;
1615         }
1616 
1617         /*
1618          * If reason state is restarter_str_service_request and
1619          * restarter_actions/auxiliary_fmri property is set with a valid fmri,
1620          * copy the fmri to restarter/auxiliary_fmri so svcs -x can use.
1621          */
1622         if (reason == restarter_str_service_request &&
1623             libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &scf_inst) == 0) {
1624                 if (restarter_inst_validate_ractions_aux_fmri(scf_inst) == 0) {
1625                         if (restarter_inst_set_aux_fmri(scf_inst))
1626                                 log_framework(LOG_DEBUG, "%s: "
1627                                     "restarter_inst_set_aux_fmri failed: ",
1628                                     rip->ri_i.i_fmri);
1629                 } else {
1630                         log_framework(LOG_DEBUG, "%s: "
1631                             "restarter_inst_validate_ractions_aux_fmri "
1632                             "failed: ", rip->ri_i.i_fmri);
1633 
1634                         if (restarter_inst_reset_aux_fmri(scf_inst))
1635                                 log_framework(LOG_DEBUG, "%s: "
1636                                     "restarter_inst_reset_aux_fmri failed: ",
1637                                     rip->ri_i.i_fmri);
1638                 }
1639                 scf_instance_destroy(scf_inst);
1640         }
1641 
1642         if (immediate || !instance_started(rip)) {
1643                 if (rip->ri_i.i_primary_ctid != 0) {
1644                         rip->ri_m_inst = safe_scf_instance_create(h);
1645                         rip->ri_mi_deleted = B_FALSE;
1646 
1647                         libscf_reget_instance(rip);
1648                         method_remove_contract(rip, B_TRUE, B_TRUE);
1649 
1650                         scf_instance_destroy(rip->ri_m_inst);
1651                 }
1652 
1653                 (void) restarter_instance_update_states(h, rip,
1654                     RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, RERR_RESTART,
1655                     reason);
1656                 return;
1657         }
1658 
1659         (void) restarter_instance_update_states(h, rip, rip->ri_i.i_state,
1660             RESTARTER_STATE_MAINT, RERR_NONE, reason);
1661 
1662         log_transition(rip, MAINT_REQUESTED);
1663 
1664         info = startd_zalloc(sizeof (*info));
1665         info->sf_id = rip->ri_id;
1666         info->sf_method_type = METHOD_STOP;
1667         info->sf_event_type = RERR_RESTART;
1668         info->sf_reason = reason;
1669         rip->ri_method_thread = startd_thread_create(method_thread, info);
1670 }
1671 
1672 static void
1673 refresh_instance(scf_handle_t *h, restarter_inst_t *rip)
1674 {
1675         scf_instance_t *inst;
1676         scf_snapshot_t *snap;
1677         fork_info_t *info;
1678         int r;
1679 
1680         assert(MUTEX_HELD(&rip->ri_lock));
1681 
1682         log_instance(rip, B_TRUE, "Rereading configuration.");
1683         log_framework(LOG_DEBUG, "%s: rereading configuration.\n",
1684             rip->ri_i.i_fmri);
1685 
1686 rep_retry:
1687         r = libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst);
1688         switch (r) {
1689         case 0:
1690                 break;
1691 
1692         case ECONNABORTED:
1693                 libscf_handle_rebind(h);
1694                 goto rep_retry;
1695 
1696         case ENOENT:
1697                 /* Must have been deleted. */
1698                 return;
1699 
1700         case EINVAL:
1701         case ENOTSUP:
1702         default:
1703                 bad_error("libscf_fmri_get_instance", r);
1704         }
1705 
1706         snap = libscf_get_running_snapshot(inst);
1707 
1708         r = libscf_get_startd_properties(inst, snap, &rip->ri_flags,
1709             &rip->ri_utmpx_prefix);
1710         switch (r) {
1711         case 0:
1712                 log_framework(LOG_DEBUG, "%s is a %s-style service\n",
1713                     rip->ri_i.i_fmri, service_style(rip->ri_flags));
1714                 break;
1715 
1716         case ECONNABORTED:
1717                 scf_instance_destroy(inst);
1718                 scf_snapshot_destroy(snap);
1719                 libscf_handle_rebind(h);
1720                 goto rep_retry;
1721 
1722         case ECANCELED:
1723         case ENOENT:
1724                 /* Succeed in anticipation of REMOVE_INSTANCE. */
1725                 break;
1726 
1727         default:
1728                 bad_error("libscf_get_startd_properties", r);
1729         }
1730 
1731         if (instance_started(rip)) {
1732                 /* Refresh does not change the state. */
1733                 (void) restarter_instance_update_states(h, rip,
1734                     rip->ri_i.i_state, rip->ri_i.i_state, RERR_NONE,
1735                     restarter_str_refresh);
1736 
1737                 info = startd_zalloc(sizeof (*info));
1738                 info->sf_id = rip->ri_id;
1739                 info->sf_method_type = METHOD_REFRESH;
1740                 info->sf_event_type = RERR_REFRESH;
1741                 info->sf_reason = NULL;
1742 
1743                 assert(rip->ri_method_thread == 0);
1744                 rip->ri_method_thread =
1745                     startd_thread_create(method_thread, info);
1746         }
1747 
1748         scf_snapshot_destroy(snap);
1749         scf_instance_destroy(inst);
1750 }
1751 
1752 const char *event_names[] = { "INVALID", "ADD_INSTANCE", "REMOVE_INSTANCE",
1753         "ENABLE", "DISABLE", "ADMIN_DEGRADED", "ADMIN_REFRESH",
1754         "ADMIN_RESTART", "ADMIN_MAINT_OFF", "ADMIN_MAINT_ON",
1755         "ADMIN_MAINT_ON_IMMEDIATE", "STOP", "START", "DEPENDENCY_CYCLE",
1756         "INVALID_DEPENDENCY", "ADMIN_DISABLE", "STOP_RESET"
1757 };
1758 
1759 /*
1760  * void *restarter_process_events()
1761  *
1762  *   Called in a separate thread to process the events on an instance's
1763  *   queue.  Empties the queue completely, and tries to keep the thread
1764  *   around for a little while after the queue is empty to save on
1765  *   startup costs.
1766  */
1767 static void *
1768 restarter_process_events(void *arg)
1769 {
1770         scf_handle_t *h;
1771         restarter_instance_qentry_t *event;
1772         restarter_inst_t *rip;
1773         char *fmri = (char *)arg;
1774         struct timespec to;
1775 
1776         (void) pthread_setname_np(pthread_self(), "restarter_process_events");
1777 
1778         assert(fmri != NULL);
1779 
1780         h = libscf_handle_create_bound_loop();
1781 
1782         /* grab the queue lock */
1783         rip = inst_lookup_queue(fmri);
1784         if (rip == NULL)
1785                 goto out;
1786 
1787 again:
1788 
1789         while ((event = uu_list_first(rip->ri_queue)) != NULL) {
1790                 restarter_inst_t *inst;
1791 
1792                 /* drop the queue lock */
1793                 MUTEX_UNLOCK(&rip->ri_queue_lock);
1794 
1795                 /*
1796                  * Grab the inst lock -- this waits until any outstanding
1797                  * method finishes running.
1798                  */
1799                 inst = inst_lookup_by_name(fmri);
1800                 if (inst == NULL) {
1801                         /* Getting deleted in the middle isn't an error. */
1802                         goto cont;
1803                 }
1804 
1805                 assert(instance_in_transition(inst) == 0);
1806 
1807                 /* process the event */
1808                 switch (event->riq_type) {
1809                 case RESTARTER_EVENT_TYPE_ENABLE:
1810                 case RESTARTER_EVENT_TYPE_DISABLE:
1811                         (void) enable_inst(h, inst, event);
1812                         break;
1813 
1814                 case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
1815                         if (enable_inst(h, inst, event) == 0)
1816                                 reset_start_times(inst);
1817                         break;
1818 
1819                 case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
1820                         restarter_delete_inst(inst);
1821                         inst = NULL;
1822                         goto cont;
1823 
1824                 case RESTARTER_EVENT_TYPE_STOP_RESET:
1825                         reset_start_times(inst);
1826                         /* FALLTHROUGH */
1827                 case RESTARTER_EVENT_TYPE_STOP:
1828                         (void) stop_instance(h, inst, RSTOP_DEPENDENCY);
1829                         break;
1830 
1831                 case RESTARTER_EVENT_TYPE_START:
1832                         start_instance(h, inst, event->riq_reason);
1833                         break;
1834 
1835                 case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
1836                         maintain_instance(h, inst, 0,
1837                             restarter_str_dependency_cycle);
1838                         break;
1839 
1840                 case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
1841                         maintain_instance(h, inst, 0,
1842                             restarter_str_invalid_dependency);
1843                         break;
1844 
1845                 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
1846                         if (event_from_tty(h, inst) == 0)
1847                                 maintain_instance(h, inst, 0,
1848                                     restarter_str_service_request);
1849                         else
1850                                 maintain_instance(h, inst, 0,
1851                                     restarter_str_administrative_request);
1852                         break;
1853 
1854                 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
1855                         if (event_from_tty(h, inst) == 0)
1856                                 maintain_instance(h, inst, 1,
1857                                     restarter_str_service_request);
1858                         else
1859                                 maintain_instance(h, inst, 1,
1860                                     restarter_str_administrative_request);
1861                         break;
1862 
1863                 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
1864                         unmaintain_instance(h, inst, RUNMAINT_CLEAR);
1865                         reset_start_times(inst);
1866                         break;
1867 
1868                 case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
1869                         refresh_instance(h, inst);
1870                         break;
1871 
1872                 case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
1873                         log_framework(LOG_WARNING, "Restarter: "
1874                             "%s command (for %s) unimplemented.\n",
1875                             event_names[event->riq_type], inst->ri_i.i_fmri);
1876                         break;
1877 
1878                 case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
1879                         if (!instance_started(inst)) {
1880                                 log_framework(LOG_DEBUG, "Restarter: "
1881                                     "Not restarting %s; not running.\n",
1882                                     inst->ri_i.i_fmri);
1883                         } else {
1884                                 /*
1885                                  * Stop the instance.  If it can be restarted,
1886                                  * the graph engine will send a new event.
1887                                  */
1888                                 if (restart_dump(h, inst)) {
1889                                         (void) contract_kill(
1890                                             inst->ri_i.i_primary_ctid, SIGABRT,
1891                                             inst->ri_i.i_fmri);
1892                                 } else if (stop_instance(h, inst,
1893                                     RSTOP_RESTART) == 0) {
1894                                         reset_start_times(inst);
1895                                 }
1896                         }
1897                         break;
1898 
1899                 case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
1900                 default:
1901 #ifndef NDEBUG
1902                         uu_warn("%s:%d: Bad restarter event %d.  "
1903                             "Aborting.\n", __FILE__, __LINE__, event->riq_type);
1904 #endif
1905                         abort();
1906                 }
1907 
1908                 assert(inst != NULL);
1909                 MUTEX_UNLOCK(&inst->ri_lock);
1910 
1911 cont:
1912                 /* grab the queue lock */
1913                 rip = inst_lookup_queue(fmri);
1914                 if (rip == NULL)
1915                         goto out;
1916 
1917                 /* delete the event */
1918                 uu_list_remove(rip->ri_queue, event);
1919                 startd_free(event, sizeof (restarter_instance_qentry_t));
1920         }
1921 
1922         assert(rip != NULL);
1923 
1924         /*
1925          * Try to preserve the thread for a little while for future use.
1926          */
1927         to.tv_sec = 3;
1928         to.tv_nsec = 0;
1929         (void) pthread_cond_reltimedwait_np(&rip->ri_queue_cv,
1930             &rip->ri_queue_lock, &to);
1931 
1932         if (uu_list_first(rip->ri_queue) != NULL)
1933                 goto again;
1934 
1935         rip->ri_queue_thread = 0;
1936         MUTEX_UNLOCK(&rip->ri_queue_lock);
1937 
1938 out:
1939         (void) scf_handle_unbind(h);
1940         scf_handle_destroy(h);
1941         free(fmri);
1942         return (NULL);
1943 }
1944 
1945 static int
1946 is_admin_event(restarter_event_type_t t)
1947 {
1948         switch (t) {
1949         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
1950         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
1951         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
1952         case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
1953         case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
1954         case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
1955                 return (1);
1956         default:
1957                 return (0);
1958         }
1959 }
1960 
1961 static void
1962 restarter_queue_event(restarter_inst_t *ri, restarter_protocol_event_t *e)
1963 {
1964         restarter_instance_qentry_t *qe;
1965         int r;
1966 
1967         assert(MUTEX_HELD(&ri->ri_queue_lock));
1968         assert(!MUTEX_HELD(&ri->ri_lock));
1969 
1970         qe = startd_zalloc(sizeof (restarter_instance_qentry_t));
1971         qe->riq_type = e->rpe_type;
1972         qe->riq_reason = e->rpe_reason;
1973 
1974         uu_list_node_init(qe, &qe->riq_link, restarter_queue_pool);
1975         r = uu_list_insert_before(ri->ri_queue, NULL, qe);
1976         assert(r == 0);
1977 }
1978 
1979 /*
1980  * void *restarter_event_thread()
1981  *
1982  *  Handle incoming graph events by placing them on a per-instance
1983  *  queue.  We can't lock the main part of the instance structure, so
1984  *  just modify the seprarately locked event queue portion.
1985  */
1986 /*ARGSUSED*/
1987 static void *
1988 restarter_event_thread(void *unused)
1989 {
1990         scf_handle_t *h;
1991 
1992         (void) pthread_setname_np(pthread_self(), "restarter_event");
1993 
1994         /*
1995          * This is a new thread, and thus, gets its own handle
1996          * to the repository.
1997          */
1998         h = libscf_handle_create_bound_loop();
1999 
2000         MUTEX_LOCK(&ru->restarter_update_lock);
2001 
2002         /*CONSTCOND*/
2003         while (1) {
2004                 restarter_protocol_event_t *e;
2005 
2006                 while (ru->restarter_update_wakeup == 0)
2007                         (void) pthread_cond_wait(&ru->restarter_update_cv,
2008                             &ru->restarter_update_lock);
2009 
2010                 ru->restarter_update_wakeup = 0;
2011 
2012                 while ((e = restarter_event_dequeue()) != NULL) {
2013                         restarter_inst_t *rip;
2014                         char *fmri;
2015 
2016                         MUTEX_UNLOCK(&ru->restarter_update_lock);
2017 
2018                         /*
2019                          * ADD_INSTANCE is special: there's likely no
2020                          * instance structure yet, so we need to handle the
2021                          * addition synchronously.
2022                          */
2023                         switch (e->rpe_type) {
2024                         case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
2025                                 if (restarter_insert_inst(h, e->rpe_inst) != 0)
2026                                         log_error(LOG_INFO, "Restarter: "
2027                                             "Could not add %s.\n", e->rpe_inst);
2028 
2029                                 MUTEX_LOCK(&st->st_load_lock);
2030                                 if (--st->st_load_instances == 0)
2031                                         (void) pthread_cond_broadcast(
2032                                             &st->st_load_cv);
2033                                 MUTEX_UNLOCK(&st->st_load_lock);
2034 
2035                                 goto nolookup;
2036                         }
2037 
2038                         /*
2039                          * Lookup the instance, locking only the event queue.
2040                          * Can't grab ri_lock here because it might be held
2041                          * by a long-running method.
2042                          */
2043                         rip = inst_lookup_queue(e->rpe_inst);
2044                         if (rip == NULL) {
2045                                 log_error(LOG_INFO, "Restarter: "
2046                                     "Ignoring %s command for unknown service "
2047                                     "%s.\n", event_names[e->rpe_type],
2048                                     e->rpe_inst);
2049                                 goto nolookup;
2050                         }
2051 
2052                         /* Keep ADMIN events from filling up the queue. */
2053                         if (is_admin_event(e->rpe_type) &&
2054                             uu_list_numnodes(rip->ri_queue) >
2055                             RINST_QUEUE_THRESHOLD) {
2056                                 MUTEX_UNLOCK(&rip->ri_queue_lock);
2057                                 log_instance(rip, B_TRUE, "Instance event "
2058                                     "queue overflow.  Dropping administrative "
2059                                     "request.");
2060                                 log_framework(LOG_DEBUG, "%s: Instance event "
2061                                     "queue overflow.  Dropping administrative "
2062                                     "request.\n", rip->ri_i.i_fmri);
2063                                 goto nolookup;
2064                         }
2065 
2066                         /* Now add the event to the instance queue. */
2067                         restarter_queue_event(rip, e);
2068 
2069                         if (rip->ri_queue_thread == 0) {
2070                                 /*
2071                                  * Start a thread if one isn't already
2072                                  * running.
2073                                  */
2074                                 fmri = safe_strdup(e->rpe_inst);
2075                                 rip->ri_queue_thread =  startd_thread_create(
2076                                     restarter_process_events, (void *)fmri);
2077                         } else {
2078                                 /*
2079                                  * Signal the existing thread that there's
2080                                  * a new event.
2081                                  */
2082                                 (void) pthread_cond_broadcast(
2083                                     &rip->ri_queue_cv);
2084                         }
2085 
2086                         MUTEX_UNLOCK(&rip->ri_queue_lock);
2087 nolookup:
2088                         restarter_event_release(e);
2089 
2090                         MUTEX_LOCK(&ru->restarter_update_lock);
2091                 }
2092         }
2093 
2094         /*
2095          * Unreachable for now -- there's currently no graceful cleanup
2096          * called on exit().
2097          */
2098         (void) scf_handle_unbind(h);
2099         scf_handle_destroy(h);
2100         return (NULL);
2101 }
2102 
2103 static restarter_inst_t *
2104 contract_to_inst(ctid_t ctid)
2105 {
2106         restarter_inst_t *inst;
2107         int id;
2108 
2109         id = lookup_inst_by_contract(ctid);
2110         if (id == -1)
2111                 return (NULL);
2112 
2113         inst = inst_lookup_by_id(id);
2114         if (inst != NULL) {
2115                 /*
2116                  * Since ri_lock isn't held by the contract id lookup, this
2117                  * instance may have been restarted and now be in a new
2118                  * contract, making the old contract no longer valid for this
2119                  * instance.
2120                  */
2121                 if (ctid != inst->ri_i.i_primary_ctid) {
2122                         MUTEX_UNLOCK(&inst->ri_lock);
2123                         inst = NULL;
2124                 }
2125         }
2126         return (inst);
2127 }
2128 
2129 /*
2130  * void contract_action()
2131  *   Take action on contract events.
2132  */
2133 static void
2134 contract_action(scf_handle_t *h, restarter_inst_t *inst, ctid_t id,
2135     uint32_t type)
2136 {
2137         const char *fmri = inst->ri_i.i_fmri;
2138 
2139         assert(MUTEX_HELD(&inst->ri_lock));
2140 
2141         /*
2142          * If startd has stopped this contract, there is no need to
2143          * stop it again.
2144          */
2145         if (inst->ri_i.i_primary_ctid > 0 &&
2146             inst->ri_i.i_primary_ctid_stopped)
2147                 return;
2148 
2149         if ((type & (CT_PR_EV_EMPTY | CT_PR_EV_CORE | CT_PR_EV_SIGNAL
2150             | CT_PR_EV_HWERR)) == 0) {
2151                 /*
2152                  * There shouldn't be other events, since that's not how we set
2153                  * the terms. Thus, just log an error and drive on.
2154                  */
2155                 log_framework(LOG_NOTICE,
2156                     "%s: contract %ld received unexpected critical event "
2157                     "(%d)\n", fmri, id, type);
2158                 return;
2159         }
2160 
2161         assert(instance_in_transition(inst) == 0);
2162 
2163         if (instance_is_wait_style(inst)) {
2164                 /*
2165                  * We ignore all events; if they impact the
2166                  * process we're monitoring, then the
2167                  * wait_thread will stop the instance.
2168                  */
2169                 log_framework(LOG_DEBUG,
2170                     "%s: ignoring contract event on wait-style service\n",
2171                     fmri);
2172         } else {
2173                 /*
2174                  * A CT_PR_EV_EMPTY event is an RSTOP_EXIT request.
2175                  */
2176                 switch (type) {
2177                 case CT_PR_EV_EMPTY:
2178                         (void) stop_instance(h, inst, RSTOP_EXIT);
2179                         break;
2180                 case CT_PR_EV_CORE:
2181                         (void) stop_instance(h, inst, RSTOP_CORE);
2182                         break;
2183                 case CT_PR_EV_SIGNAL:
2184                         (void) stop_instance(h, inst, RSTOP_SIGNAL);
2185                         break;
2186                 case CT_PR_EV_HWERR:
2187                         (void) stop_instance(h, inst, RSTOP_HWERR);
2188                         break;
2189                 }
2190         }
2191 }
2192 
2193 /*
2194  * void *restarter_contract_event_thread(void *)
2195  *   Listens to the process contract bundle for critical events, taking action
2196  *   on events from contracts we know we are responsible for.
2197  */
2198 /*ARGSUSED*/
2199 static void *
2200 restarter_contracts_event_thread(void *unused)
2201 {
2202         int fd, err;
2203         scf_handle_t *local_handle;
2204 
2205         (void) pthread_setname_np(pthread_self(), "restarter_contracts_event");
2206 
2207         /*
2208          * Await graph load completion.  That is, stop here, until we've scanned
2209          * the repository for contract - instance associations.
2210          */
2211         MUTEX_LOCK(&st->st_load_lock);
2212         while (!(st->st_load_complete && st->st_load_instances == 0))
2213                 (void) pthread_cond_wait(&st->st_load_cv, &st->st_load_lock);
2214         MUTEX_UNLOCK(&st->st_load_lock);
2215 
2216         /*
2217          * This is a new thread, and thus, gets its own handle
2218          * to the repository.
2219          */
2220         if ((local_handle = libscf_handle_create_bound(SCF_VERSION)) == NULL)
2221                 uu_die("Unable to bind a new repository handle: %s\n",
2222                     scf_strerror(scf_error()));
2223 
2224         fd = open64(CTFS_ROOT "/process/pbundle", O_RDONLY);
2225         if (fd == -1)
2226                 uu_die("process bundle open failed");
2227 
2228         /*
2229          * Make sure we get all events (including those generated by configd
2230          * before this thread was started).
2231          */
2232         err = ct_event_reset(fd);
2233         assert(err == 0);
2234 
2235         for (;;) {
2236                 int efd, sfd;
2237                 ct_evthdl_t ev;
2238                 uint32_t type;
2239                 ctevid_t evid;
2240                 ct_stathdl_t status;
2241                 ctid_t ctid;
2242                 restarter_inst_t *inst;
2243                 uint64_t cookie;
2244 
2245                 if (err = ct_event_read_critical(fd, &ev)) {
2246                         log_error(LOG_WARNING,
2247                             "Error reading next contract event: %s",
2248                             strerror(err));
2249                         continue;
2250                 }
2251 
2252                 evid = ct_event_get_evid(ev);
2253                 ctid = ct_event_get_ctid(ev);
2254                 type = ct_event_get_type(ev);
2255 
2256                 /* Fetch cookie. */
2257                 if ((sfd = contract_open(ctid, "process", "status", O_RDONLY))
2258                     < 0) {
2259                         ct_event_free(ev);
2260                         continue;
2261                 }
2262 
2263                 if (err = ct_status_read(sfd, CTD_COMMON, &status)) {
2264                         log_framework(LOG_WARNING, "Could not get status for "
2265                             "contract %ld: %s\n", ctid, strerror(err));
2266 
2267                         startd_close(sfd);
2268                         ct_event_free(ev);
2269                         continue;
2270                 }
2271 
2272                 cookie = ct_status_get_cookie(status);
2273 
2274                 log_framework(LOG_DEBUG, "Received event %d for ctid %ld "
2275                     "cookie %lld\n", type, ctid, cookie);
2276 
2277                 ct_status_free(status);
2278 
2279                 startd_close(sfd);
2280 
2281                 /*
2282                  * svc.configd(1M) restart handling performed by the
2283                  * fork_configd_thread.  We don't acknowledge, as that thread
2284                  * will do so.
2285                  */
2286                 if (cookie == CONFIGD_COOKIE) {
2287                         ct_event_free(ev);
2288                         continue;
2289                 }
2290 
2291                 inst = NULL;
2292                 if (storing_contract != 0 &&
2293                     (inst = contract_to_inst(ctid)) == NULL) {
2294                         /*
2295                          * This can happen for two reasons:
2296                          * - method_run() has not yet stored the
2297                          *    the contract into the internal hash table.
2298                          * - we receive an EMPTY event for an abandoned
2299                          *    contract.
2300                          * If there is any contract in the process of
2301                          * being stored into the hash table then re-read
2302                          * the event later.
2303                          */
2304                         log_framework(LOG_DEBUG,
2305                             "Reset event %d for unknown "
2306                             "contract id %ld\n", type, ctid);
2307 
2308                         /* don't go too fast */
2309                         (void) poll(NULL, 0, 100);
2310 
2311                         (void) ct_event_reset(fd);
2312                         ct_event_free(ev);
2313                         continue;
2314                 }
2315 
2316                 /*
2317                  * Do not call contract_to_inst() again if first
2318                  * call succeeded.
2319                  */
2320                 if (inst == NULL)
2321                         inst = contract_to_inst(ctid);
2322                 if (inst == NULL) {
2323                         /*
2324                          * This can happen if we receive an EMPTY
2325                          * event for an abandoned contract.
2326                          */
2327                         log_framework(LOG_DEBUG,
2328                             "Received event %d for unknown contract id "
2329                             "%ld\n", type, ctid);
2330                 } else {
2331                         log_framework(LOG_DEBUG,
2332                             "Received event %d for contract id "
2333                             "%ld (%s)\n", type, ctid,
2334                             inst->ri_i.i_fmri);
2335 
2336                         contract_action(local_handle, inst, ctid, type);
2337 
2338                         MUTEX_UNLOCK(&inst->ri_lock);
2339                 }
2340 
2341                 efd = contract_open(ct_event_get_ctid(ev), "process", "ctl",
2342                     O_WRONLY);
2343                 if (efd != -1) {
2344                         (void) ct_ctl_ack(efd, evid);
2345                         startd_close(efd);
2346                 }
2347 
2348                 ct_event_free(ev);
2349 
2350         }
2351 
2352         /*NOTREACHED*/
2353         return (NULL);
2354 }
2355 
2356 /*
2357  * Timeout queue, processed by restarter_timeouts_event_thread().
2358  */
2359 timeout_queue_t *timeouts;
2360 static uu_list_pool_t *timeout_pool;
2361 
2362 typedef struct timeout_update {
2363         pthread_mutex_t         tu_lock;
2364         pthread_cond_t          tu_cv;
2365         int                     tu_wakeup;
2366 } timeout_update_t;
2367 
2368 timeout_update_t *tu;
2369 
2370 static const char *timeout_ovr_svcs[] = {
2371         "svc:/system/manifest-import:default",
2372         "svc:/network/initial:default",
2373         "svc:/network/service:default",
2374         "svc:/system/rmtmpfiles:default",
2375         "svc:/network/loopback:default",
2376         "svc:/network/physical:default",
2377         "svc:/system/device/local:default",
2378         "svc:/system/filesystem/usr:default",
2379         "svc:/system/filesystem/minimal:default",
2380         "svc:/system/filesystem/local:default",
2381         NULL
2382 };
2383 
2384 int
2385 is_timeout_ovr(restarter_inst_t *inst)
2386 {
2387         int i;
2388 
2389         for (i = 0; timeout_ovr_svcs[i] != NULL; ++i) {
2390                 if (strcmp(inst->ri_i.i_fmri, timeout_ovr_svcs[i]) == 0) {
2391                         log_instance(inst, B_TRUE, "Timeout override by "
2392                             "svc.startd.  Using infinite timeout.");
2393                         return (1);
2394                 }
2395         }
2396 
2397         return (0);
2398 }
2399 
2400 /*ARGSUSED*/
2401 static int
2402 timeout_compare(const void *lc_arg, const void *rc_arg, void *private)
2403 {
2404         hrtime_t t1 = ((const timeout_entry_t *)lc_arg)->te_timeout;
2405         hrtime_t t2 = ((const timeout_entry_t *)rc_arg)->te_timeout;
2406 
2407         if (t1 > t2)
2408                 return (1);
2409         else if (t1 < t2)
2410                 return (-1);
2411         return (0);
2412 }
2413 
2414 void
2415 timeout_init()
2416 {
2417         timeouts = startd_zalloc(sizeof (timeout_queue_t));
2418 
2419         (void) pthread_mutex_init(&timeouts->tq_lock, &mutex_attrs);
2420 
2421         timeout_pool = startd_list_pool_create("timeouts",
2422             sizeof (timeout_entry_t), offsetof(timeout_entry_t, te_link),
2423             timeout_compare, UU_LIST_POOL_DEBUG);
2424         assert(timeout_pool != NULL);
2425 
2426         timeouts->tq_list = startd_list_create(timeout_pool,
2427             timeouts, UU_LIST_SORTED);
2428         assert(timeouts->tq_list != NULL);
2429 
2430         tu = startd_zalloc(sizeof (timeout_update_t));
2431         (void) pthread_cond_init(&tu->tu_cv, NULL);
2432         (void) pthread_mutex_init(&tu->tu_lock, &mutex_attrs);
2433 }
2434 
2435 void
2436 timeout_insert(restarter_inst_t *inst, ctid_t cid, uint64_t timeout_sec)
2437 {
2438         hrtime_t now, timeout;
2439         timeout_entry_t *entry;
2440         uu_list_index_t idx;
2441 
2442         assert(MUTEX_HELD(&inst->ri_lock));
2443 
2444         now = gethrtime();
2445 
2446         /*
2447          * If we overflow LLONG_MAX, we're never timing out anyways, so
2448          * just return.
2449          */
2450         if (timeout_sec >= (LLONG_MAX - now) / 1000000000LL) {
2451                 log_instance(inst, B_TRUE, "timeout_seconds too large, "
2452                     "treating as infinite.");
2453                 return;
2454         }
2455 
2456         /* hrtime is in nanoseconds. Convert timeout_sec. */
2457         timeout = now + (timeout_sec * 1000000000LL);
2458 
2459         entry = startd_alloc(sizeof (timeout_entry_t));
2460         entry->te_timeout = timeout;
2461         entry->te_ctid = cid;
2462         entry->te_fmri = safe_strdup(inst->ri_i.i_fmri);
2463         entry->te_logstem = safe_strdup(inst->ri_logstem);
2464         entry->te_fired = 0;
2465         /* Insert the calculated timeout time onto the queue. */
2466         MUTEX_LOCK(&timeouts->tq_lock);
2467         (void) uu_list_find(timeouts->tq_list, entry, NULL, &idx);
2468         uu_list_node_init(entry, &entry->te_link, timeout_pool);
2469         uu_list_insert(timeouts->tq_list, entry, idx);
2470         MUTEX_UNLOCK(&timeouts->tq_lock);
2471 
2472         assert(inst->ri_timeout == NULL);
2473         inst->ri_timeout = entry;
2474 
2475         MUTEX_LOCK(&tu->tu_lock);
2476         tu->tu_wakeup = 1;
2477         (void) pthread_cond_broadcast(&tu->tu_cv);
2478         MUTEX_UNLOCK(&tu->tu_lock);
2479 }
2480 
2481 
2482 void
2483 timeout_remove(restarter_inst_t *inst, ctid_t cid)
2484 {
2485         assert(MUTEX_HELD(&inst->ri_lock));
2486 
2487         if (inst->ri_timeout == NULL)
2488                 return;
2489 
2490         assert(inst->ri_timeout->te_ctid == cid);
2491 
2492         MUTEX_LOCK(&timeouts->tq_lock);
2493         uu_list_remove(timeouts->tq_list, inst->ri_timeout);
2494         MUTEX_UNLOCK(&timeouts->tq_lock);
2495 
2496         free(inst->ri_timeout->te_fmri);
2497         free(inst->ri_timeout->te_logstem);
2498         startd_free(inst->ri_timeout, sizeof (timeout_entry_t));
2499         inst->ri_timeout = NULL;
2500 }
2501 
2502 static int
2503 timeout_now()
2504 {
2505         timeout_entry_t *e;
2506         hrtime_t now;
2507         int ret;
2508 
2509         now = gethrtime();
2510 
2511         /*
2512          * Walk through the (sorted) timeouts list.  While the timeout
2513          * at the head of the list is <= the current time, kill the
2514          * method.
2515          */
2516         MUTEX_LOCK(&timeouts->tq_lock);
2517 
2518         for (e = uu_list_first(timeouts->tq_list);
2519             e != NULL && e->te_timeout <= now;
2520             e = uu_list_next(timeouts->tq_list, e)) {
2521                 log_framework(LOG_WARNING, "%s: Method or service exit timed "
2522                     "out.  Killing contract %ld.\n", e->te_fmri, e->te_ctid);
2523                 log_instance_fmri(e->te_fmri, e->te_logstem, B_TRUE,
2524                     "Method or service exit timed out.  Killing contract %ld.",
2525                     e->te_ctid);
2526                 e->te_fired = 1;
2527                 (void) contract_kill(e->te_ctid, SIGKILL, e->te_fmri);
2528         }
2529 
2530         if (uu_list_numnodes(timeouts->tq_list) > 0)
2531                 ret = 0;
2532         else
2533                 ret = -1;
2534 
2535         MUTEX_UNLOCK(&timeouts->tq_lock);
2536 
2537         return (ret);
2538 }
2539 
2540 /*
2541  * void *restarter_timeouts_event_thread(void *)
2542  *   Responsible for monitoring the method timeouts.  This thread must
2543  *   be started before any methods are called.
2544  */
2545 /*ARGSUSED*/
2546 static void *
2547 restarter_timeouts_event_thread(void *unused)
2548 {
2549         /*
2550          * Timeouts are entered on a priority queue, which is processed by
2551          * this thread.  As timeouts are specified in seconds, we'll do
2552          * the necessary processing every second, as long as the queue
2553          * is not empty.
2554          */
2555 
2556         (void) pthread_setname_np(pthread_self(), "restarter_timeouts_event");
2557 
2558         /*CONSTCOND*/
2559         while (1) {
2560                 /*
2561                  * As long as the timeout list isn't empty, process it
2562                  * every second.
2563                  */
2564                 if (timeout_now() == 0) {
2565                         (void) sleep(1);
2566                         continue;
2567                 }
2568 
2569                 /* The list is empty, wait until we have more timeouts. */
2570                 MUTEX_LOCK(&tu->tu_lock);
2571 
2572                 while (tu->tu_wakeup == 0)
2573                         (void) pthread_cond_wait(&tu->tu_cv, &tu->tu_lock);
2574 
2575                 tu->tu_wakeup = 0;
2576                 MUTEX_UNLOCK(&tu->tu_lock);
2577         }
2578 
2579         return (NULL);
2580 }
2581 
2582 void
2583 restarter_start()
2584 {
2585         (void) startd_thread_create(restarter_timeouts_event_thread, NULL);
2586         (void) startd_thread_create(restarter_event_thread, NULL);
2587         (void) startd_thread_create(restarter_contracts_event_thread, NULL);
2588         (void) startd_thread_create(wait_thread, NULL);
2589 }
2590 
2591 
2592 void
2593 restarter_init()
2594 {
2595         restarter_instance_pool = startd_list_pool_create("restarter_instances",
2596             sizeof (restarter_inst_t), offsetof(restarter_inst_t,
2597             ri_link), restarter_instance_compare, UU_LIST_POOL_DEBUG);
2598         (void) memset(&instance_list, 0, sizeof (instance_list));
2599 
2600         (void) pthread_mutex_init(&instance_list.ril_lock, &mutex_attrs);
2601         instance_list.ril_instance_list = startd_list_create(
2602             restarter_instance_pool, &instance_list, UU_LIST_SORTED);
2603 
2604         restarter_queue_pool = startd_list_pool_create(
2605             "restarter_instance_queue", sizeof (restarter_instance_qentry_t),
2606             offsetof(restarter_instance_qentry_t,  riq_link), NULL,
2607             UU_LIST_POOL_DEBUG);
2608 
2609         contract_list_pool = startd_list_pool_create(
2610             "contract_list", sizeof (contract_entry_t),
2611             offsetof(contract_entry_t,  ce_link), NULL,
2612             UU_LIST_POOL_DEBUG);
2613         contract_hash_init();
2614 
2615         log_framework(LOG_DEBUG, "Initialized restarter\n");
2616 }