1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  25  */
  26 
  27 /*
  28  * restarter.c - service manipulation
  29  *
  30  * This component manages services whose restarter is svc.startd, the standard
  31  * restarter.  It translates restarter protocol events from the graph engine
  32  * into actions on processes, as a delegated restarter would do.
  33  *
  34  * The master restarter manages a number of always-running threads:
  35  *   - restarter event thread: events from the graph engine
  36  *   - timeout thread: thread to fire queued timeouts
  37  *   - contract thread: thread to handle contract events
  38  *   - wait thread: thread to handle wait-based services
  39  *
  40  * The other threads are created as-needed:
  41  *   - per-instance method threads
  42  *   - per-instance event processing threads
  43  *
  44  * The interaction of all threads must result in the following conditions
  45  * being satisfied (on a per-instance basis):
  46  *   - restarter events must be processed in order
  47  *   - method execution must be serialized
  48  *   - instance delete must be held until outstanding methods are complete
  49  *   - contract events shouldn't be processed while a method is running
  50  *   - timeouts should fire even when a method is running
  51  *
  52  * Service instances are represented by restarter_inst_t's and are kept in the
  53  * instance_list list.
  54  *
  55  * Service States
  56  *   The current state of a service instance is kept in
  57  *   restarter_inst_t->ri_i.i_state.  If transition to a new state could take
  58  *   some time, then before we effect the transition we set
  59  *   restarter_inst_t->ri_i.i_next_state to the target state, and afterwards we
  60  *   rotate i_next_state to i_state and set i_next_state to
  61  *   RESTARTER_STATE_NONE.  So usually i_next_state is _NONE when ri_lock is not
  62  *   held.  The exception is when we launch methods, which are done with
  63  *   a separate thread.  To keep any other threads from grabbing ri_lock before
  64  *   method_thread() does, we set ri_method_thread to the thread id of the
  65  *   method thread, and when it is nonzero any thread with a different thread id
  66  *   waits on ri_method_cv.
  67  *
  68  * Method execution is serialized by blocking on ri_method_cv in
  69  * inst_lookup_by_id() and waiting for a 0 value of ri_method_thread.  This
  70  * also prevents the instance structure from being deleted until all
  71  * outstanding operations such as method_thread() have finished.
  72  *
  73  * Lock ordering:
  74  *
  75  * dgraph_lock [can be held when taking:]
  76  *   utmpx_lock
  77  *   dictionary->dict_lock
  78  *   st->st_load_lock
  79  *   wait_info_lock
  80  *   ru->restarter_update_lock
  81  *     restarter_queue->rpeq_lock
  82  *   instance_list.ril_lock
  83  *     inst->ri_lock
  84  *   st->st_configd_live_lock
  85  *
  86  * instance_list.ril_lock
  87  *   graph_queue->gpeq_lock
  88  *   gu->gu_lock
  89  *   st->st_configd_live_lock
  90  *   dictionary->dict_lock
  91  *   inst->ri_lock
  92  *     graph_queue->gpeq_lock
  93  *     gu->gu_lock
  94  *     tu->tu_lock
  95  *     tq->tq_lock
  96  *     inst->ri_queue_lock
  97  *       wait_info_lock
  98  *       bp->cb_lock
  99  *     utmpx_lock
 100  *
 101  * single_user_thread_lock
 102  *   wait_info_lock
 103  *   utmpx_lock
 104  *
 105  * gu_freeze_lock
 106  *
 107  * logbuf_mutex nests inside pretty much everything.
 108  */
 109 
 110 #include <sys/contract/process.h>
 111 #include <sys/ctfs.h>
 112 #include <sys/stat.h>
 113 #include <sys/time.h>
 114 #include <sys/types.h>
 115 #include <sys/uio.h>
 116 #include <sys/wait.h>
 117 #include <assert.h>
 118 #include <errno.h>
 119 #include <fcntl.h>
 120 #include <libcontract.h>
 121 #include <libcontract_priv.h>
 122 #include <libintl.h>
 123 #include <librestart.h>
 124 #include <librestart_priv.h>
 125 #include <libuutil.h>
 126 #include <limits.h>
 127 #include <poll.h>
 128 #include <port.h>
 129 #include <pthread.h>
 130 #include <stdarg.h>
 131 #include <stdio.h>
 132 #include <strings.h>
 133 #include <unistd.h>
 134 
 135 #include "startd.h"
 136 #include "protocol.h"
 137 
 138 static uu_list_pool_t *restarter_instance_pool;
 139 static restarter_instance_list_t instance_list;
 140 
 141 static uu_list_pool_t *restarter_queue_pool;
 142 
 143 #define WT_SVC_ERR_THROTTLE     1       /* 1 sec delay for erroring wait svc */
 144 
 145 /*
 146  * Function used to reset the restart times for an instance, when
 147  * an administrative task comes along and essentially makes the times
 148  * in this array ineffective.
 149  */
 150 static void
 151 reset_start_times(restarter_inst_t *inst)
 152 {
 153         inst->ri_start_index = 0;
 154         bzero(inst->ri_start_time, sizeof (inst->ri_start_time));
 155 }
 156 
 157 /*ARGSUSED*/
 158 static int
 159 restarter_instance_compare(const void *lc_arg, const void *rc_arg,
 160     void *private)
 161 {
 162         int lc_id = ((const restarter_inst_t *)lc_arg)->ri_id;
 163         int rc_id = *(int *)rc_arg;
 164 
 165         if (lc_id > rc_id)
 166                 return (1);
 167         if (lc_id < rc_id)
 168                 return (-1);
 169         return (0);
 170 }
 171 
 172 static restarter_inst_t *
 173 inst_lookup_by_name(const char *name)
 174 {
 175         int id;
 176 
 177         id = dict_lookup_byname(name);
 178         if (id == -1)
 179                 return (NULL);
 180 
 181         return (inst_lookup_by_id(id));
 182 }
 183 
 184 restarter_inst_t *
 185 inst_lookup_by_id(int id)
 186 {
 187         restarter_inst_t *inst;
 188 
 189         MUTEX_LOCK(&instance_list.ril_lock);
 190         inst = uu_list_find(instance_list.ril_instance_list, &id, NULL, NULL);
 191         if (inst != NULL)
 192                 MUTEX_LOCK(&inst->ri_lock);
 193         MUTEX_UNLOCK(&instance_list.ril_lock);
 194 
 195         if (inst != NULL) {
 196                 while (inst->ri_method_thread != 0 &&
 197                     !pthread_equal(inst->ri_method_thread, pthread_self())) {
 198                         ++inst->ri_method_waiters;
 199                         (void) pthread_cond_wait(&inst->ri_method_cv,
 200                             &inst->ri_lock);
 201                         assert(inst->ri_method_waiters > 0);
 202                         --inst->ri_method_waiters;
 203                 }
 204         }
 205 
 206         return (inst);
 207 }
 208 
 209 static restarter_inst_t *
 210 inst_lookup_queue(const char *name)
 211 {
 212         int id;
 213         restarter_inst_t *inst;
 214 
 215         id = dict_lookup_byname(name);
 216         if (id == -1)
 217                 return (NULL);
 218 
 219         MUTEX_LOCK(&instance_list.ril_lock);
 220         inst = uu_list_find(instance_list.ril_instance_list, &id, NULL, NULL);
 221         if (inst != NULL)
 222                 MUTEX_LOCK(&inst->ri_queue_lock);
 223         MUTEX_UNLOCK(&instance_list.ril_lock);
 224 
 225         return (inst);
 226 }
 227 
 228 const char *
 229 service_style(int flags)
 230 {
 231         switch (flags & RINST_STYLE_MASK) {
 232         case RINST_CONTRACT:    return ("contract");
 233         case RINST_TRANSIENT:   return ("transient");
 234         case RINST_WAIT:        return ("wait");
 235 
 236         default:
 237 #ifndef NDEBUG
 238                 uu_warn("%s:%d: Bad flags 0x%x.\n", __FILE__, __LINE__, flags);
 239 #endif
 240                 abort();
 241                 /* NOTREACHED */
 242         }
 243 }
 244 
 245 /*
 246  * Fails with ECONNABORTED or ECANCELED.
 247  */
 248 static int
 249 check_contract(restarter_inst_t *inst, boolean_t primary,
 250     scf_instance_t *scf_inst)
 251 {
 252         ctid_t *ctidp;
 253         int fd, r;
 254 
 255         ctidp = primary ? &inst->ri_i.i_primary_ctid :
 256             &inst->ri_i.i_transient_ctid;
 257 
 258         assert(*ctidp >= 1);
 259 
 260         fd = contract_open(*ctidp, NULL, "status", O_RDONLY);
 261         if (fd >= 0) {
 262                 r = close(fd);
 263                 assert(r == 0);
 264                 return (0);
 265         }
 266 
 267         r = restarter_remove_contract(scf_inst, *ctidp, primary ?
 268             RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
 269         switch (r) {
 270         case 0:
 271         case ECONNABORTED:
 272         case ECANCELED:
 273                 *ctidp = 0;
 274                 return (r);
 275 
 276         case ENOMEM:
 277                 uu_die("Out of memory\n");
 278                 /* NOTREACHED */
 279 
 280         case EPERM:
 281                 uu_die("Insufficient privilege.\n");
 282                 /* NOTREACHED */
 283 
 284         case EACCES:
 285                 uu_die("Repository backend access denied.\n");
 286                 /* NOTREACHED */
 287 
 288         case EROFS:
 289                 log_error(LOG_INFO, "Could not remove unusable contract id %ld "
 290                     "for %s from repository.\n", *ctidp, inst->ri_i.i_fmri);
 291                 return (0);
 292 
 293         case EINVAL:
 294         case EBADF:
 295         default:
 296                 assert(0);
 297                 abort();
 298                 /* NOTREACHED */
 299         }
 300 }
 301 
 302 static int stop_instance(scf_handle_t *, restarter_inst_t *, stop_cause_t);
 303 
 304 /*
 305  * int restarter_insert_inst(scf_handle_t *, char *)
 306  *   If the inst is already in the restarter list, return its id.  If the inst
 307  *   is not in the restarter list, initialize a restarter_inst_t, initialize its
 308  *   states, insert it into the list, and return 0.
 309  *
 310  *   Fails with
 311  *     ENOENT - name is not in the repository
 312  */
 313 static int
 314 restarter_insert_inst(scf_handle_t *h, const char *name)
 315 {
 316         int id, r;
 317         restarter_inst_t *inst;
 318         uu_list_index_t idx;
 319         scf_service_t *scf_svc;
 320         scf_instance_t *scf_inst;
 321         scf_snapshot_t *snap = NULL;
 322         scf_propertygroup_t *pg;
 323         char *svc_name, *inst_name;
 324         char logfilebuf[PATH_MAX];
 325         char *c;
 326         boolean_t do_commit_states;
 327         restarter_instance_state_t state, next_state;
 328         protocol_states_t *ps;
 329         pid_t start_pid;
 330         restarter_str_t reason = restarter_str_insert_in_graph;
 331 
 332         MUTEX_LOCK(&instance_list.ril_lock);
 333 
 334         /*
 335          * We don't use inst_lookup_by_name() here because we want the lookup
 336          * & insert to be atomic.
 337          */
 338         id = dict_lookup_byname(name);
 339         if (id != -1) {
 340                 inst = uu_list_find(instance_list.ril_instance_list, &id, NULL,
 341                     &idx);
 342                 if (inst != NULL) {
 343                         MUTEX_UNLOCK(&instance_list.ril_lock);
 344                         return (0);
 345                 }
 346         }
 347 
 348         /* Allocate an instance */
 349         inst = startd_zalloc(sizeof (restarter_inst_t));
 350         inst->ri_utmpx_prefix = startd_alloc(max_scf_value_size);
 351         inst->ri_utmpx_prefix[0] = '\0';
 352 
 353         inst->ri_i.i_fmri = startd_alloc(strlen(name) + 1);
 354         (void) strcpy((char *)inst->ri_i.i_fmri, name);
 355 
 356         inst->ri_queue = startd_list_create(restarter_queue_pool, inst, 0);
 357 
 358         /*
 359          * id shouldn't be -1 since we use the same dictionary as graph.c, but
 360          * just in case.
 361          */
 362         inst->ri_id = (id != -1 ? id : dict_insert(name));
 363 
 364         special_online_hooks_get(name, &inst->ri_pre_online_hook,
 365             &inst->ri_post_online_hook, &inst->ri_post_offline_hook);
 366 
 367         scf_svc = safe_scf_service_create(h);
 368         scf_inst = safe_scf_instance_create(h);
 369         pg = safe_scf_pg_create(h);
 370         svc_name = startd_alloc(max_scf_name_size);
 371         inst_name = startd_alloc(max_scf_name_size);
 372 
 373 rep_retry:
 374         if (snap != NULL)
 375                 scf_snapshot_destroy(snap);
 376         if (inst->ri_logstem != NULL)
 377                 startd_free(inst->ri_logstem, PATH_MAX);
 378         if (inst->ri_common_name != NULL)
 379                 free(inst->ri_common_name);
 380         if (inst->ri_C_common_name != NULL)
 381                 free(inst->ri_C_common_name);
 382         snap = NULL;
 383         inst->ri_logstem = NULL;
 384         inst->ri_common_name = NULL;
 385         inst->ri_C_common_name = NULL;
 386 
 387         if (scf_handle_decode_fmri(h, name, NULL, scf_svc, scf_inst, NULL,
 388             NULL, SCF_DECODE_FMRI_EXACT) != 0) {
 389                 switch (scf_error()) {
 390                 case SCF_ERROR_CONNECTION_BROKEN:
 391                         libscf_handle_rebind(h);
 392                         goto rep_retry;
 393 
 394                 case SCF_ERROR_NOT_FOUND:
 395                         goto deleted;
 396                 }
 397 
 398                 uu_die("Can't decode FMRI %s: %s\n", name,
 399                     scf_strerror(scf_error()));
 400         }
 401 
 402         /*
 403          * If there's no running snapshot, then we execute using the editing
 404          * snapshot.  Pending snapshots will be taken later.
 405          */
 406         snap = libscf_get_running_snapshot(scf_inst);
 407 
 408         if ((scf_service_get_name(scf_svc, svc_name, max_scf_name_size) < 0) ||
 409             (scf_instance_get_name(scf_inst, inst_name, max_scf_name_size) <
 410             0)) {
 411                 switch (scf_error()) {
 412                 case SCF_ERROR_NOT_SET:
 413                         break;
 414 
 415                 case SCF_ERROR_CONNECTION_BROKEN:
 416                         libscf_handle_rebind(h);
 417                         goto rep_retry;
 418 
 419                 default:
 420                         assert(0);
 421                         abort();
 422                 }
 423 
 424                 goto deleted;
 425         }
 426 
 427         (void) snprintf(logfilebuf, PATH_MAX, "%s:%s", svc_name, inst_name);
 428         for (c = logfilebuf; *c != '\0'; c++)
 429                 if (*c == '/')
 430                         *c = '-';
 431 
 432         inst->ri_logstem = startd_alloc(PATH_MAX);
 433         (void) snprintf(inst->ri_logstem, PATH_MAX, "%s%s", logfilebuf,
 434             LOG_SUFFIX);
 435 
 436         /*
 437          * If the restarter group is missing, use uninit/none.  Otherwise,
 438          * we're probably being restarted & don't want to mess up the states
 439          * that are there.
 440          */
 441         state = RESTARTER_STATE_UNINIT;
 442         next_state = RESTARTER_STATE_NONE;
 443 
 444         r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
 445         if (r != 0) {
 446                 switch (scf_error()) {
 447                 case SCF_ERROR_CONNECTION_BROKEN:
 448                         libscf_handle_rebind(h);
 449                         goto rep_retry;
 450 
 451                 case SCF_ERROR_NOT_SET:
 452                         goto deleted;
 453 
 454                 case SCF_ERROR_NOT_FOUND:
 455                         /*
 456                          * This shouldn't happen since the graph engine should
 457                          * have initialized the state to uninitialized/none if
 458                          * there was no restarter pg.  In case somebody
 459                          * deleted it, though....
 460                          */
 461                         do_commit_states = B_TRUE;
 462                         break;
 463 
 464                 default:
 465                         assert(0);
 466                         abort();
 467                 }
 468         } else {
 469                 r = libscf_read_states(pg, &state, &next_state);
 470                 if (r != 0) {
 471                         do_commit_states = B_TRUE;
 472                 } else {
 473                         if (next_state != RESTARTER_STATE_NONE) {
 474                                 /*
 475                                  * Force next_state to _NONE since we
 476                                  * don't look for method processes.
 477                                  */
 478                                 next_state = RESTARTER_STATE_NONE;
 479                                 do_commit_states = B_TRUE;
 480                         } else {
 481                                 /*
 482                                  * The reason for transition will depend on
 483                                  * state.
 484                                  */
 485                                 if (st->st_initial == 0)
 486                                         reason = restarter_str_startd_restart;
 487                                 else if (state == RESTARTER_STATE_MAINT)
 488                                         reason = restarter_str_bad_repo_state;
 489                                 /*
 490                                  * Inform the restarter of our state without
 491                                  * changing the STIME in the repository.
 492                                  */
 493                                 ps = startd_alloc(sizeof (*ps));
 494                                 inst->ri_i.i_state = ps->ps_state = state;
 495                                 inst->ri_i.i_next_state = ps->ps_state_next =
 496                                     next_state;
 497                                 ps->ps_reason = reason;
 498 
 499                                 graph_protocol_send_event(inst->ri_i.i_fmri,
 500                                     GRAPH_UPDATE_STATE_CHANGE, ps);
 501 
 502                                 do_commit_states = B_FALSE;
 503                         }
 504                 }
 505         }
 506 
 507         switch (libscf_get_startd_properties(scf_inst, snap, &inst->ri_flags,
 508             &inst->ri_utmpx_prefix)) {
 509         case 0:
 510                 break;
 511 
 512         case ECONNABORTED:
 513                 libscf_handle_rebind(h);
 514                 goto rep_retry;
 515 
 516         case ECANCELED:
 517                 goto deleted;
 518 
 519         case ENOENT:
 520                 /*
 521                  * This is odd, because the graph engine should have required
 522                  * the general property group.  So we'll just use default
 523                  * flags in anticipation of the graph engine sending us
 524                  * REMOVE_INSTANCE when it finds out that the general property
 525                  * group has been deleted.
 526                  */
 527                 inst->ri_flags = RINST_CONTRACT;
 528                 break;
 529 
 530         default:
 531                 assert(0);
 532                 abort();
 533         }
 534 
 535         r = libscf_get_template_values(scf_inst, snap,
 536             &inst->ri_common_name, &inst->ri_C_common_name);
 537 
 538         /*
 539          * Copy our names to smaller buffers to reduce our memory footprint.
 540          */
 541         if (inst->ri_common_name != NULL) {
 542                 char *tmp = safe_strdup(inst->ri_common_name);
 543                 startd_free(inst->ri_common_name, max_scf_value_size);
 544                 inst->ri_common_name = tmp;
 545         }
 546 
 547         if (inst->ri_C_common_name != NULL) {
 548                 char *tmp = safe_strdup(inst->ri_C_common_name);
 549                 startd_free(inst->ri_C_common_name, max_scf_value_size);
 550                 inst->ri_C_common_name = tmp;
 551         }
 552 
 553         switch (r) {
 554         case 0:
 555                 break;
 556 
 557         case ECONNABORTED:
 558                 libscf_handle_rebind(h);
 559                 goto rep_retry;
 560 
 561         case ECANCELED:
 562                 goto deleted;
 563 
 564         case ECHILD:
 565         case ENOENT:
 566                 break;
 567 
 568         default:
 569                 assert(0);
 570                 abort();
 571         }
 572 
 573         switch (libscf_read_method_ids(h, scf_inst, inst->ri_i.i_fmri,
 574             &inst->ri_i.i_primary_ctid, &inst->ri_i.i_transient_ctid,
 575             &start_pid)) {
 576         case 0:
 577                 break;
 578 
 579         case ECONNABORTED:
 580                 libscf_handle_rebind(h);
 581                 goto rep_retry;
 582 
 583         case ECANCELED:
 584                 goto deleted;
 585 
 586         default:
 587                 assert(0);
 588                 abort();
 589         }
 590 
 591         if (inst->ri_i.i_primary_ctid >= 1) {
 592                 contract_hash_store(inst->ri_i.i_primary_ctid, inst->ri_id);
 593 
 594                 switch (check_contract(inst, B_TRUE, scf_inst)) {
 595                 case 0:
 596                         break;
 597 
 598                 case ECONNABORTED:
 599                         libscf_handle_rebind(h);
 600                         goto rep_retry;
 601 
 602                 case ECANCELED:
 603                         goto deleted;
 604 
 605                 default:
 606                         assert(0);
 607                         abort();
 608                 }
 609         }
 610 
 611         if (inst->ri_i.i_transient_ctid >= 1) {
 612                 switch (check_contract(inst, B_FALSE, scf_inst)) {
 613                 case 0:
 614                         break;
 615 
 616                 case ECONNABORTED:
 617                         libscf_handle_rebind(h);
 618                         goto rep_retry;
 619 
 620                 case ECANCELED:
 621                         goto deleted;
 622 
 623                 default:
 624                         assert(0);
 625                         abort();
 626                 }
 627         }
 628 
 629         /* No more failures we live through, so add it to the list. */
 630         (void) pthread_mutex_init(&inst->ri_lock, &mutex_attrs);
 631         (void) pthread_mutex_init(&inst->ri_queue_lock, &mutex_attrs);
 632         MUTEX_LOCK(&inst->ri_lock);
 633         MUTEX_LOCK(&inst->ri_queue_lock);
 634 
 635         (void) pthread_cond_init(&inst->ri_method_cv, NULL);
 636 
 637         uu_list_node_init(inst, &inst->ri_link, restarter_instance_pool);
 638         uu_list_insert(instance_list.ril_instance_list, inst, idx);
 639         MUTEX_UNLOCK(&instance_list.ril_lock);
 640 
 641         if (start_pid != -1 &&
 642             (inst->ri_flags & RINST_STYLE_MASK) == RINST_WAIT) {
 643                 int ret;
 644                 ret = wait_register(start_pid, inst->ri_i.i_fmri, 0, 1);
 645                 if (ret == -1) {
 646                         /*
 647                          * Implication:  if we can't reregister the
 648                          * instance, we will start another one.  Two
 649                          * instances may or may not result in a resource
 650                          * conflict.
 651                          */
 652                         log_error(LOG_WARNING,
 653                             "%s: couldn't reregister %ld for wait\n",
 654                             inst->ri_i.i_fmri, start_pid);
 655                 } else if (ret == 1) {
 656                         /*
 657                          * Leading PID has exited.
 658                          */
 659                         (void) stop_instance(h, inst, RSTOP_EXIT);
 660                 }
 661         }
 662 
 663 
 664         scf_pg_destroy(pg);
 665 
 666         if (do_commit_states)
 667                 (void) restarter_instance_update_states(h, inst, state,
 668                     next_state, RERR_NONE, reason);
 669 
 670         log_framework(LOG_DEBUG, "%s is a %s-style service\n", name,
 671             service_style(inst->ri_flags));
 672 
 673         MUTEX_UNLOCK(&inst->ri_queue_lock);
 674         MUTEX_UNLOCK(&inst->ri_lock);
 675 
 676         startd_free(svc_name, max_scf_name_size);
 677         startd_free(inst_name, max_scf_name_size);
 678         scf_snapshot_destroy(snap);
 679         scf_instance_destroy(scf_inst);
 680         scf_service_destroy(scf_svc);
 681 
 682         log_framework(LOG_DEBUG, "%s: inserted instance into restarter list\n",
 683             name);
 684 
 685         return (0);
 686 
 687 deleted:
 688         MUTEX_UNLOCK(&instance_list.ril_lock);
 689         startd_free(inst_name, max_scf_name_size);
 690         startd_free(svc_name, max_scf_name_size);
 691         if (snap != NULL)
 692                 scf_snapshot_destroy(snap);
 693         scf_pg_destroy(pg);
 694         scf_instance_destroy(scf_inst);
 695         scf_service_destroy(scf_svc);
 696         startd_free((void *)inst->ri_i.i_fmri, strlen(inst->ri_i.i_fmri) + 1);
 697         uu_list_destroy(inst->ri_queue);
 698         if (inst->ri_logstem != NULL)
 699                 startd_free(inst->ri_logstem, PATH_MAX);
 700         if (inst->ri_common_name != NULL)
 701                 free(inst->ri_common_name);
 702         if (inst->ri_C_common_name != NULL)
 703                 free(inst->ri_C_common_name);
 704         startd_free(inst->ri_utmpx_prefix, max_scf_value_size);
 705         startd_free(inst, sizeof (restarter_inst_t));
 706         return (ENOENT);
 707 }
 708 
 709 static void
 710 restarter_delete_inst(restarter_inst_t *ri)
 711 {
 712         int id;
 713         restarter_inst_t *rip;
 714         void *cookie = NULL;
 715         restarter_instance_qentry_t *e;
 716 
 717         assert(MUTEX_HELD(&ri->ri_lock));
 718 
 719         /*
 720          * Must drop the instance lock so we can pick up the instance_list
 721          * lock & remove the instance.
 722          */
 723         id = ri->ri_id;
 724         MUTEX_UNLOCK(&ri->ri_lock);
 725 
 726         MUTEX_LOCK(&instance_list.ril_lock);
 727 
 728         rip = uu_list_find(instance_list.ril_instance_list, &id, NULL, NULL);
 729         if (rip == NULL) {
 730                 MUTEX_UNLOCK(&instance_list.ril_lock);
 731                 return;
 732         }
 733 
 734         assert(ri == rip);
 735 
 736         uu_list_remove(instance_list.ril_instance_list, ri);
 737 
 738         log_framework(LOG_DEBUG, "%s: deleted instance from restarter list\n",
 739             ri->ri_i.i_fmri);
 740 
 741         MUTEX_UNLOCK(&instance_list.ril_lock);
 742 
 743         /*
 744          * We can lock the instance without holding the instance_list lock
 745          * since we removed the instance from the list.
 746          */
 747         MUTEX_LOCK(&ri->ri_lock);
 748         MUTEX_LOCK(&ri->ri_queue_lock);
 749 
 750         if (ri->ri_i.i_primary_ctid >= 1)
 751                 contract_hash_remove(ri->ri_i.i_primary_ctid);
 752 
 753         while (ri->ri_method_thread != 0 || ri->ri_method_waiters > 0)
 754                 (void) pthread_cond_wait(&ri->ri_method_cv, &ri->ri_lock);
 755 
 756         while ((e = uu_list_teardown(ri->ri_queue, &cookie)) != NULL)
 757                 startd_free(e, sizeof (*e));
 758         uu_list_destroy(ri->ri_queue);
 759 
 760         startd_free((void *)ri->ri_i.i_fmri, strlen(ri->ri_i.i_fmri) + 1);
 761         startd_free(ri->ri_logstem, PATH_MAX);
 762         if (ri->ri_common_name != NULL)
 763                 free(ri->ri_common_name);
 764         if (ri->ri_C_common_name != NULL)
 765                 free(ri->ri_C_common_name);
 766         startd_free(ri->ri_utmpx_prefix, max_scf_value_size);
 767         (void) pthread_mutex_destroy(&ri->ri_lock);
 768         (void) pthread_mutex_destroy(&ri->ri_queue_lock);
 769         startd_free(ri, sizeof (restarter_inst_t));
 770 }
 771 
 772 /*
 773  * instance_is_wait_style()
 774  *
 775  *   Returns 1 if the given instance is a "wait-style" service instance.
 776  */
 777 int
 778 instance_is_wait_style(restarter_inst_t *inst)
 779 {
 780         assert(MUTEX_HELD(&inst->ri_lock));
 781         return ((inst->ri_flags & RINST_STYLE_MASK) == RINST_WAIT);
 782 }
 783 
 784 /*
 785  * instance_is_transient_style()
 786  *
 787  *   Returns 1 if the given instance is a transient service instance.
 788  */
 789 int
 790 instance_is_transient_style(restarter_inst_t *inst)
 791 {
 792         assert(MUTEX_HELD(&inst->ri_lock));
 793         return ((inst->ri_flags & RINST_STYLE_MASK) == RINST_TRANSIENT);
 794 }
 795 
 796 /*
 797  * instance_in_transition()
 798  * Returns 1 if instance is in transition, 0 if not
 799  */
 800 int
 801 instance_in_transition(restarter_inst_t *inst)
 802 {
 803         assert(MUTEX_HELD(&inst->ri_lock));
 804         if (inst->ri_i.i_next_state == RESTARTER_STATE_NONE)
 805                 return (0);
 806         return (1);
 807 }
 808 
 809 /*
 810  * returns 1 if instance is already started, 0 if not
 811  */
 812 static int
 813 instance_started(restarter_inst_t *inst)
 814 {
 815         int ret;
 816 
 817         assert(MUTEX_HELD(&inst->ri_lock));
 818 
 819         if (inst->ri_i.i_state == RESTARTER_STATE_ONLINE ||
 820             inst->ri_i.i_state == RESTARTER_STATE_DEGRADED)
 821                 ret = 1;
 822         else
 823                 ret = 0;
 824 
 825         return (ret);
 826 }
 827 
 828 /*
 829  * Returns
 830  *   0 - success
 831  *   ECONNRESET - success, but h was rebound
 832  */
 833 int
 834 restarter_instance_update_states(scf_handle_t *h, restarter_inst_t *ri,
 835     restarter_instance_state_t new_state,
 836     restarter_instance_state_t new_state_next, restarter_error_t err,
 837     restarter_str_t reason)
 838 {
 839         protocol_states_t *states;
 840         int e;
 841         uint_t retry_count = 0, msecs = ALLOC_DELAY;
 842         boolean_t rebound = B_FALSE;
 843         int prev_state_online;
 844         int state_online;
 845 
 846         assert(MUTEX_HELD(&ri->ri_lock));
 847 
 848         prev_state_online = instance_started(ri);
 849 
 850 retry:
 851         e = _restarter_commit_states(h, &ri->ri_i, new_state, new_state_next,
 852             restarter_get_str_short(reason));
 853         switch (e) {
 854         case 0:
 855                 break;
 856 
 857         case ENOMEM:
 858                 ++retry_count;
 859                 if (retry_count < ALLOC_RETRY) {
 860                         (void) poll(NULL, 0, msecs);
 861                         msecs *= ALLOC_DELAY_MULT;
 862                         goto retry;
 863                 }
 864 
 865                 /* Like startd_alloc(). */
 866                 uu_die("Insufficient memory.\n");
 867                 /* NOTREACHED */
 868 
 869         case ECONNABORTED:
 870                 libscf_handle_rebind(h);
 871                 rebound = B_TRUE;
 872                 goto retry;
 873 
 874         case EPERM:
 875         case EACCES:
 876         case EROFS:
 877                 log_error(LOG_NOTICE, "Could not commit state change for %s "
 878                     "to repository: %s.\n", ri->ri_i.i_fmri, strerror(e));
 879                 /* FALLTHROUGH */
 880 
 881         case ENOENT:
 882                 ri->ri_i.i_state = new_state;
 883                 ri->ri_i.i_next_state = new_state_next;
 884                 break;
 885 
 886         case EINVAL:
 887         default:
 888                 bad_error("_restarter_commit_states", e);
 889         }
 890 
 891         states = startd_alloc(sizeof (protocol_states_t));
 892         states->ps_state = new_state;
 893         states->ps_state_next = new_state_next;
 894         states->ps_err = err;
 895         states->ps_reason = reason;
 896         graph_protocol_send_event(ri->ri_i.i_fmri, GRAPH_UPDATE_STATE_CHANGE,
 897             (void *)states);
 898 
 899         state_online = instance_started(ri);
 900 
 901         if (prev_state_online && !state_online)
 902                 ri->ri_post_offline_hook();
 903         else if (!prev_state_online && state_online)
 904                 ri->ri_post_online_hook();
 905 
 906         return (rebound ? ECONNRESET : 0);
 907 }
 908 
 909 void
 910 restarter_mark_pending_snapshot(const char *fmri, uint_t flag)
 911 {
 912         restarter_inst_t *inst;
 913 
 914         assert(flag == RINST_RETAKE_RUNNING || flag == RINST_RETAKE_START);
 915 
 916         inst = inst_lookup_by_name(fmri);
 917         if (inst == NULL)
 918                 return;
 919 
 920         inst->ri_flags |= flag;
 921 
 922         MUTEX_UNLOCK(&inst->ri_lock);
 923 }
 924 
 925 static void
 926 restarter_take_pending_snapshots(scf_handle_t *h)
 927 {
 928         restarter_inst_t *inst;
 929         int r;
 930 
 931         MUTEX_LOCK(&instance_list.ril_lock);
 932 
 933         for (inst = uu_list_first(instance_list.ril_instance_list);
 934             inst != NULL;
 935             inst = uu_list_next(instance_list.ril_instance_list, inst)) {
 936                 const char *fmri;
 937                 scf_instance_t *sinst = NULL;
 938 
 939                 MUTEX_LOCK(&inst->ri_lock);
 940 
 941                 /*
 942                  * This is where we'd check inst->ri_method_thread and if it
 943                  * were nonzero we'd wait in anticipation of another thread
 944                  * executing a method for inst.  Doing so with the instance_list
 945                  * locked, though, leads to deadlock.  Since taking a snapshot
 946                  * during that window won't hurt anything, we'll just continue.
 947                  */
 948 
 949                 fmri = inst->ri_i.i_fmri;
 950 
 951                 if (inst->ri_flags & RINST_RETAKE_RUNNING) {
 952                         scf_snapshot_t *rsnap;
 953 
 954                         (void) libscf_fmri_get_instance(h, fmri, &sinst);
 955 
 956                         rsnap = libscf_get_or_make_running_snapshot(sinst,
 957                             fmri, B_FALSE);
 958 
 959                         scf_instance_destroy(sinst);
 960 
 961                         if (rsnap != NULL)
 962                                 inst->ri_flags &= ~RINST_RETAKE_RUNNING;
 963 
 964                         scf_snapshot_destroy(rsnap);
 965                 }
 966 
 967                 if (inst->ri_flags & RINST_RETAKE_START) {
 968                         switch (r = libscf_snapshots_poststart(h, fmri,
 969                             B_FALSE)) {
 970                         case 0:
 971                         case ENOENT:
 972                                 inst->ri_flags &= ~RINST_RETAKE_START;
 973                                 break;
 974 
 975                         case ECONNABORTED:
 976                                 break;
 977 
 978                         case EACCES:
 979                         default:
 980                                 bad_error("libscf_snapshots_poststart", r);
 981                         }
 982                 }
 983 
 984                 MUTEX_UNLOCK(&inst->ri_lock);
 985         }
 986 
 987         MUTEX_UNLOCK(&instance_list.ril_lock);
 988 }
 989 
 990 /* ARGSUSED */
 991 void *
 992 restarter_post_fsminimal_thread(void *unused)
 993 {
 994         scf_handle_t *h;
 995         int r;
 996 
 997         h = libscf_handle_create_bound_loop();
 998 
 999         for (;;) {
1000                 r = libscf_create_self(h);
1001                 if (r == 0)
1002                         break;
1003 
1004                 assert(r == ECONNABORTED);
1005                 libscf_handle_rebind(h);
1006         }
1007 
1008         restarter_take_pending_snapshots(h);
1009 
1010         (void) scf_handle_unbind(h);
1011         scf_handle_destroy(h);
1012 
1013         return (NULL);
1014 }
1015 
1016 /*
1017  * int stop_instance()
1018  *
1019  *   Stop the instance identified by the instance given as the second argument,
1020  *   for the cause stated.
1021  *
1022  *   Returns
1023  *     0 - success
1024  *     -1 - inst is in transition
1025  */
1026 static int
1027 stop_instance(scf_handle_t *local_handle, restarter_inst_t *inst,
1028     stop_cause_t cause)
1029 {
1030         fork_info_t *info;
1031         const char *cp;
1032         int err;
1033         restarter_error_t re;
1034         restarter_str_t reason;
1035         restarter_instance_state_t new_state;
1036 
1037         assert(MUTEX_HELD(&inst->ri_lock));
1038         assert(inst->ri_method_thread == 0);
1039 
1040         switch (cause) {
1041         case RSTOP_EXIT:
1042                 re = RERR_RESTART;
1043                 reason = restarter_str_ct_ev_exit;
1044                 cp = "all processes in service exited";
1045                 break;
1046         case RSTOP_ERR_CFG:
1047                 re = RERR_FAULT;
1048                 reason = restarter_str_method_failed;
1049                 cp = "service exited with a configuration error";
1050                 break;
1051         case RSTOP_ERR_EXIT:
1052                 re = RERR_RESTART;
1053                 reason = restarter_str_ct_ev_exit;
1054                 cp = "service exited with an error";
1055                 break;
1056         case RSTOP_CORE:
1057                 re = RERR_FAULT;
1058                 reason = restarter_str_ct_ev_core;
1059                 cp = "process dumped core";
1060                 break;
1061         case RSTOP_SIGNAL:
1062                 re = RERR_FAULT;
1063                 reason = restarter_str_ct_ev_signal;
1064                 cp = "process received fatal signal from outside the service";
1065                 break;
1066         case RSTOP_HWERR:
1067                 re = RERR_FAULT;
1068                 reason = restarter_str_ct_ev_hwerr;
1069                 cp = "process killed due to uncorrectable hardware error";
1070                 break;
1071         case RSTOP_DEPENDENCY:
1072                 re = RERR_RESTART;
1073                 reason = restarter_str_dependency_activity;
1074                 cp = "dependency activity requires stop";
1075                 break;
1076         case RSTOP_DISABLE:
1077                 re = RERR_RESTART;
1078                 reason = restarter_str_disable_request;
1079                 cp = "service disabled";
1080                 break;
1081         case RSTOP_RESTART:
1082                 re = RERR_RESTART;
1083                 reason = restarter_str_restart_request;
1084                 cp = "service restarting";
1085                 break;
1086         default:
1087 #ifndef NDEBUG
1088                 (void) fprintf(stderr, "Unknown cause %d at %s:%d.\n",
1089                     cause, __FILE__, __LINE__);
1090 #endif
1091                 abort();
1092         }
1093 
1094         /* Services in the disabled and maintenance state are ignored */
1095         if (inst->ri_i.i_state == RESTARTER_STATE_MAINT ||
1096             inst->ri_i.i_state == RESTARTER_STATE_DISABLED) {
1097                 log_framework(LOG_DEBUG,
1098                     "%s: stop_instance -> is maint/disabled\n",
1099                     inst->ri_i.i_fmri);
1100                 return (0);
1101         }
1102 
1103         /* Already stopped instances are left alone */
1104         if (instance_started(inst) == 0) {
1105                 log_framework(LOG_DEBUG, "Restarter: %s is already stopped.\n",
1106                     inst->ri_i.i_fmri);
1107                 return (0);
1108         }
1109 
1110         if (instance_in_transition(inst)) {
1111                 /* requeue event by returning -1 */
1112                 log_framework(LOG_DEBUG,
1113                     "Restarter: Not stopping %s, in transition.\n",
1114                     inst->ri_i.i_fmri);
1115                 return (-1);
1116         }
1117 
1118         log_instance(inst, B_TRUE, "Stopping because %s.", cp);
1119 
1120         log_framework(re == RERR_FAULT ? LOG_INFO : LOG_DEBUG,
1121             "%s: Instance stopping because %s.\n", inst->ri_i.i_fmri, cp);
1122 
1123         if (instance_is_wait_style(inst) &&
1124             (cause == RSTOP_EXIT ||
1125             cause == RSTOP_ERR_CFG ||
1126             cause == RSTOP_ERR_EXIT)) {
1127                 /*
1128                  * No need to stop instance, as child has exited; remove
1129                  * contract and move the instance to the offline state.
1130                  */
1131                 switch (err = restarter_instance_update_states(local_handle,
1132                     inst, inst->ri_i.i_state, RESTARTER_STATE_OFFLINE, re,
1133                     reason)) {
1134                 case 0:
1135                 case ECONNRESET:
1136                         break;
1137 
1138                 default:
1139                         bad_error("restarter_instance_update_states", err);
1140                 }
1141 
1142                 if (cause == RSTOP_ERR_EXIT) {
1143                         /*
1144                          * The RSTOP_ERR_EXIT cause is set via the
1145                          * wait_thread -> wait_remove code path when we have
1146                          * a "wait" style svc that exited with an error. If
1147                          * the svc is failing too quickly, we throttle it so
1148                          * that we don't restart it more than once/second.
1149                          * Since we know we're running in the wait thread its
1150                          * ok to throttle it right here.
1151                          */
1152                         (void) update_fault_count(inst, FAULT_COUNT_INCR);
1153                         if (method_rate_critical(inst)) {
1154                                 log_instance(inst, B_TRUE, "Failing too "
1155                                     "quickly, throttling.");
1156                                 (void) sleep(WT_SVC_ERR_THROTTLE);
1157                         }
1158                 } else {
1159                         (void) update_fault_count(inst, FAULT_COUNT_RESET);
1160                         reset_start_times(inst);
1161                 }
1162 
1163                 if (inst->ri_i.i_primary_ctid != 0) {
1164                         inst->ri_m_inst =
1165                             safe_scf_instance_create(local_handle);
1166                         inst->ri_mi_deleted = B_FALSE;
1167 
1168                         libscf_reget_instance(inst);
1169                         method_remove_contract(inst, B_TRUE, B_TRUE);
1170 
1171                         scf_instance_destroy(inst->ri_m_inst);
1172                         inst->ri_m_inst = NULL;
1173                 }
1174 
1175                 switch (err = restarter_instance_update_states(local_handle,
1176                     inst, inst->ri_i.i_next_state, RESTARTER_STATE_NONE, re,
1177                     reason)) {
1178                 case 0:
1179                 case ECONNRESET:
1180                         break;
1181 
1182                 default:
1183                         bad_error("restarter_instance_update_states", err);
1184                 }
1185 
1186                 if (cause != RSTOP_ERR_CFG)
1187                         return (0);
1188         } else if (instance_is_wait_style(inst) && re == RERR_RESTART) {
1189                 /*
1190                  * Stopping a wait service through means other than the pid
1191                  * exiting should keep wait_thread() from restarting the
1192                  * service, by removing it from the wait list.
1193                  * We cannot remove it right now otherwise the process will
1194                  * end up <defunct> so mark it to be ignored.
1195                  */
1196                 wait_ignore_by_fmri(inst->ri_i.i_fmri);
1197         }
1198 
1199         /*
1200          * There are some configuration errors which we cannot detect until we
1201          * try to run the method.  For example, see exec_method() where the
1202          * restarter_set_method_context() call can return SMF_EXIT_ERR_CONFIG
1203          * in several cases. If this happens for a "wait-style" svc,
1204          * wait_remove() sets the cause as RSTOP_ERR_CFG so that we can detect
1205          * the configuration error and go into maintenance, even though it is
1206          * a "wait-style" svc.
1207          */
1208         if (cause == RSTOP_ERR_CFG)
1209                 new_state = RESTARTER_STATE_MAINT;
1210         else
1211                 new_state = inst->ri_i.i_enabled ?
1212                     RESTARTER_STATE_OFFLINE : RESTARTER_STATE_DISABLED;
1213 
1214         switch (err = restarter_instance_update_states(local_handle, inst,
1215             inst->ri_i.i_state, new_state, RERR_NONE, reason)) {
1216         case 0:
1217         case ECONNRESET:
1218                 break;
1219 
1220         default:
1221                 bad_error("restarter_instance_update_states", err);
1222         }
1223 
1224         info = startd_zalloc(sizeof (fork_info_t));
1225 
1226         info->sf_id = inst->ri_id;
1227         info->sf_method_type = METHOD_STOP;
1228         info->sf_event_type = re;
1229         info->sf_reason = reason;
1230         inst->ri_method_thread = startd_thread_create(method_thread, info);
1231 
1232         return (0);
1233 }
1234 
1235 /*
1236  * Returns
1237  *   ENOENT - fmri is not in instance_list
1238  *   0 - success
1239  *   ECONNRESET - success, though handle was rebound
1240  *   -1 - instance is in transition
1241  */
1242 int
1243 stop_instance_fmri(scf_handle_t *h, const char *fmri, uint_t flags)
1244 {
1245         restarter_inst_t *rip;
1246         int r;
1247 
1248         rip = inst_lookup_by_name(fmri);
1249         if (rip == NULL)
1250                 return (ENOENT);
1251 
1252         r = stop_instance(h, rip, flags);
1253 
1254         MUTEX_UNLOCK(&rip->ri_lock);
1255 
1256         return (r);
1257 }
1258 
1259 static void
1260 unmaintain_instance(scf_handle_t *h, restarter_inst_t *rip,
1261     unmaint_cause_t cause)
1262 {
1263         ctid_t ctid;
1264         scf_instance_t *inst;
1265         int r;
1266         uint_t tries = 0, msecs = ALLOC_DELAY;
1267         const char *cp;
1268         restarter_str_t reason;
1269 
1270         assert(MUTEX_HELD(&rip->ri_lock));
1271 
1272         if (rip->ri_i.i_state != RESTARTER_STATE_MAINT) {
1273                 log_error(LOG_DEBUG, "Restarter: "
1274                     "Ignoring maintenance off command because %s is not in the "
1275                     "maintenance state.\n", rip->ri_i.i_fmri);
1276                 return;
1277         }
1278 
1279         switch (cause) {
1280         case RUNMAINT_CLEAR:
1281                 cp = "clear requested";
1282                 reason = restarter_str_clear_request;
1283                 break;
1284         case RUNMAINT_DISABLE:
1285                 cp = "disable requested";
1286                 reason = restarter_str_disable_request;
1287                 break;
1288         default:
1289 #ifndef NDEBUG
1290                 (void) fprintf(stderr, "Uncaught case for %d at %s:%d.\n",
1291                     cause, __FILE__, __LINE__);
1292 #endif
1293                 abort();
1294         }
1295 
1296         log_instance(rip, B_TRUE, "Leaving maintenance because %s.",
1297             cp);
1298         log_framework(LOG_DEBUG, "%s: Instance leaving maintenance because "
1299             "%s.\n", rip->ri_i.i_fmri, cp);
1300 
1301         (void) restarter_instance_update_states(h, rip, RESTARTER_STATE_UNINIT,
1302             RESTARTER_STATE_NONE, RERR_RESTART, reason);
1303 
1304         /*
1305          * If we did ADMIN_MAINT_ON_IMMEDIATE, then there might still be
1306          * a primary contract.
1307          */
1308         if (rip->ri_i.i_primary_ctid == 0)
1309                 return;
1310 
1311         ctid = rip->ri_i.i_primary_ctid;
1312         contract_abandon(ctid);
1313         rip->ri_i.i_primary_ctid = 0;
1314 
1315 rep_retry:
1316         switch (r = libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst)) {
1317         case 0:
1318                 break;
1319 
1320         case ECONNABORTED:
1321                 libscf_handle_rebind(h);
1322                 goto rep_retry;
1323 
1324         case ENOENT:
1325                 /* Must have been deleted. */
1326                 return;
1327 
1328         case EINVAL:
1329         case ENOTSUP:
1330         default:
1331                 bad_error("libscf_handle_rebind", r);
1332         }
1333 
1334 again:
1335         r = restarter_remove_contract(inst, ctid, RESTARTER_CONTRACT_PRIMARY);
1336         switch (r) {
1337         case 0:
1338                 break;
1339 
1340         case ENOMEM:
1341                 ++tries;
1342                 if (tries < ALLOC_RETRY) {
1343                         (void) poll(NULL, 0, msecs);
1344                         msecs *= ALLOC_DELAY_MULT;
1345                         goto again;
1346                 }
1347 
1348                 uu_die("Insufficient memory.\n");
1349                 /* NOTREACHED */
1350 
1351         case ECONNABORTED:
1352                 scf_instance_destroy(inst);
1353                 libscf_handle_rebind(h);
1354                 goto rep_retry;
1355 
1356         case ECANCELED:
1357                 break;
1358 
1359         case EPERM:
1360         case EACCES:
1361         case EROFS:
1362                 log_error(LOG_INFO,
1363                     "Could not remove contract id %lu for %s (%s).\n", ctid,
1364                     rip->ri_i.i_fmri, strerror(r));
1365                 break;
1366 
1367         case EINVAL:
1368         case EBADF:
1369         default:
1370                 bad_error("restarter_remove_contract", r);
1371         }
1372 
1373         scf_instance_destroy(inst);
1374 }
1375 
1376 /*
1377  * enable_inst()
1378  *   Set inst->ri_i.i_enabled.  Expects 'e' to be _ENABLE, _DISABLE, or
1379  *   _ADMIN_DISABLE.  If the event is _ENABLE and inst is uninitialized or
1380  *   disabled, move it to offline.  If the event is _DISABLE or
1381  *   _ADMIN_DISABLE, make sure inst will move to disabled.
1382  *
1383  *   Returns
1384  *     0 - success
1385  *     ECONNRESET - h was rebound
1386  */
1387 static int
1388 enable_inst(scf_handle_t *h, restarter_inst_t *inst,
1389     restarter_instance_qentry_t *riq)
1390 {
1391         restarter_instance_state_t state;
1392         restarter_event_type_t e = riq->riq_type;
1393         restarter_str_t reason = restarter_str_per_configuration;
1394         int r;
1395 
1396         assert(MUTEX_HELD(&inst->ri_lock));
1397         assert(e == RESTARTER_EVENT_TYPE_ADMIN_DISABLE ||
1398             e == RESTARTER_EVENT_TYPE_DISABLE ||
1399             e == RESTARTER_EVENT_TYPE_ENABLE);
1400         assert(instance_in_transition(inst) == 0);
1401 
1402         state = inst->ri_i.i_state;
1403 
1404         if (e == RESTARTER_EVENT_TYPE_ENABLE) {
1405                 inst->ri_i.i_enabled = 1;
1406 
1407                 if (state == RESTARTER_STATE_UNINIT ||
1408                     state == RESTARTER_STATE_DISABLED) {
1409                         /*
1410                          * B_FALSE: Don't log an error if the log_instance()
1411                          * fails because it will fail on the miniroot before
1412                          * install-discovery runs.
1413                          */
1414                         log_instance(inst, B_FALSE, "Enabled.");
1415                         log_framework(LOG_DEBUG, "%s: Instance enabled.\n",
1416                             inst->ri_i.i_fmri);
1417 
1418                         /*
1419                          * If we are coming from DISABLED, it was obviously an
1420                          * enable request. If we are coming from UNINIT, it may
1421                          * have been a sevice in MAINT that was cleared.
1422                          */
1423                         if (riq->riq_reason == restarter_str_clear_request)
1424                                 reason = restarter_str_clear_request;
1425                         else if (state == RESTARTER_STATE_DISABLED)
1426                                 reason = restarter_str_enable_request;
1427                         (void) restarter_instance_update_states(h, inst,
1428                             RESTARTER_STATE_OFFLINE, RESTARTER_STATE_NONE,
1429                             RERR_NONE, reason);
1430                 } else {
1431                         log_framework(LOG_DEBUG, "Restarter: "
1432                             "Not changing state of %s for enable command.\n",
1433                             inst->ri_i.i_fmri);
1434                 }
1435         } else {
1436                 inst->ri_i.i_enabled = 0;
1437 
1438                 switch (state) {
1439                 case RESTARTER_STATE_ONLINE:
1440                 case RESTARTER_STATE_DEGRADED:
1441                         r = stop_instance(h, inst, RSTOP_DISABLE);
1442                         return (r == ECONNRESET ? 0 : r);
1443 
1444                 case RESTARTER_STATE_OFFLINE:
1445                 case RESTARTER_STATE_UNINIT:
1446                         if (inst->ri_i.i_primary_ctid != 0) {
1447                                 inst->ri_m_inst = safe_scf_instance_create(h);
1448                                 inst->ri_mi_deleted = B_FALSE;
1449 
1450                                 libscf_reget_instance(inst);
1451                                 method_remove_contract(inst, B_TRUE, B_TRUE);
1452 
1453                                 scf_instance_destroy(inst->ri_m_inst);
1454                         }
1455                         /* B_FALSE: See log_instance(..., "Enabled."); above */
1456                         log_instance(inst, B_FALSE, "Disabled.");
1457                         log_framework(LOG_DEBUG, "%s: Instance disabled.\n",
1458                             inst->ri_i.i_fmri);
1459 
1460                         /*
1461                          * If we are coming from OFFLINE, it was obviously a
1462                          * disable request. But if we are coming from
1463                          * UNINIT, it may have been a disable request for a
1464                          * service in MAINT.
1465                          */
1466                         if (riq->riq_reason == restarter_str_disable_request ||
1467                             state == RESTARTER_STATE_OFFLINE)
1468                                 reason = restarter_str_disable_request;
1469                         (void) restarter_instance_update_states(h, inst,
1470                             RESTARTER_STATE_DISABLED, RESTARTER_STATE_NONE,
1471                             RERR_RESTART, reason);
1472                         return (0);
1473 
1474                 case RESTARTER_STATE_DISABLED:
1475                         break;
1476 
1477                 case RESTARTER_STATE_MAINT:
1478                         /*
1479                          * We only want to pull the instance out of maintenance
1480                          * if the disable is on adminstrative request.  The
1481                          * graph engine sends _DISABLE events whenever a
1482                          * service isn't in the disabled state, and we don't
1483                          * want to pull the service out of maintenance if,
1484                          * for example, it is there due to a dependency cycle.
1485                          */
1486                         if (e == RESTARTER_EVENT_TYPE_ADMIN_DISABLE)
1487                                 unmaintain_instance(h, inst, RUNMAINT_DISABLE);
1488                         break;
1489 
1490                 default:
1491 #ifndef NDEBUG
1492                         (void) fprintf(stderr, "Restarter instance %s has "
1493                             "unknown state %d.\n", inst->ri_i.i_fmri, state);
1494 #endif
1495                         abort();
1496                 }
1497         }
1498 
1499         return (0);
1500 }
1501 
1502 static void
1503 start_instance(scf_handle_t *local_handle, restarter_inst_t *inst,
1504     int32_t reason)
1505 {
1506         fork_info_t *info;
1507         restarter_str_t new_reason;
1508 
1509         assert(MUTEX_HELD(&inst->ri_lock));
1510         assert(instance_in_transition(inst) == 0);
1511         assert(inst->ri_method_thread == 0);
1512 
1513         log_framework(LOG_DEBUG, "%s: trying to start instance\n",
1514             inst->ri_i.i_fmri);
1515 
1516         /*
1517          * We want to keep the original reason for restarts and clear actions
1518          */
1519         switch (reason) {
1520         case restarter_str_restart_request:
1521         case restarter_str_clear_request:
1522                 new_reason = reason;
1523                 break;
1524         default:
1525                 new_reason = restarter_str_dependencies_satisfied;
1526         }
1527 
1528         /* Services in the disabled and maintenance state are ignored */
1529         if (inst->ri_i.i_state == RESTARTER_STATE_MAINT ||
1530             inst->ri_i.i_state == RESTARTER_STATE_DISABLED ||
1531             inst->ri_i.i_enabled == 0) {
1532                 log_framework(LOG_DEBUG,
1533                     "%s: start_instance -> is maint/disabled\n",
1534                     inst->ri_i.i_fmri);
1535                 return;
1536         }
1537 
1538         /* Already started instances are left alone */
1539         if (instance_started(inst) == 1) {
1540                 log_framework(LOG_DEBUG,
1541                     "%s: start_instance -> is already started\n",
1542                     inst->ri_i.i_fmri);
1543                 return;
1544         }
1545 
1546         log_framework(LOG_DEBUG, "%s: starting instance.\n", inst->ri_i.i_fmri);
1547 
1548         (void) restarter_instance_update_states(local_handle, inst,
1549             inst->ri_i.i_state, RESTARTER_STATE_ONLINE, RERR_NONE, new_reason);
1550 
1551         info = startd_zalloc(sizeof (fork_info_t));
1552 
1553         info->sf_id = inst->ri_id;
1554         info->sf_method_type = METHOD_START;
1555         info->sf_event_type = RERR_NONE;
1556         info->sf_reason = new_reason;
1557         inst->ri_method_thread = startd_thread_create(method_thread, info);
1558 }
1559 
1560 static int
1561 event_from_tty(scf_handle_t *h, restarter_inst_t *rip)
1562 {
1563         scf_instance_t *inst;
1564         int ret = 0;
1565 
1566         if (libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst))
1567                 return (-1);
1568 
1569         ret = restarter_inst_ractions_from_tty(inst);
1570 
1571         scf_instance_destroy(inst);
1572         return (ret);
1573 }
1574 
1575 static boolean_t
1576 restart_dump(scf_handle_t *h, restarter_inst_t *rip)
1577 {
1578         scf_instance_t *inst;
1579         boolean_t ret = B_FALSE;
1580 
1581         if (libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst))
1582                 return (-1);
1583 
1584         if (restarter_inst_dump(inst) == 1)
1585                 ret = B_TRUE;
1586 
1587         scf_instance_destroy(inst);
1588         return (ret);
1589 }
1590 
1591 static void
1592 maintain_instance(scf_handle_t *h, restarter_inst_t *rip, int immediate,
1593     restarter_str_t reason)
1594 {
1595         fork_info_t *info;
1596         scf_instance_t *scf_inst = NULL;
1597 
1598         assert(MUTEX_HELD(&rip->ri_lock));
1599         assert(reason != restarter_str_none);
1600         assert(rip->ri_method_thread == 0);
1601 
1602         log_instance(rip, B_TRUE, "Stopping for maintenance due to %s.",
1603             restarter_get_str_short(reason));
1604         log_framework(LOG_DEBUG, "%s: stopping for maintenance due to %s.\n",
1605             rip->ri_i.i_fmri, restarter_get_str_short(reason));
1606 
1607         /* Services in the maintenance state are ignored */
1608         if (rip->ri_i.i_state == RESTARTER_STATE_MAINT) {
1609                 log_framework(LOG_DEBUG,
1610                     "%s: maintain_instance -> is already in maintenance\n",
1611                     rip->ri_i.i_fmri);
1612                 return;
1613         }
1614 
1615         /*
1616          * If reason state is restarter_str_service_request and
1617          * restarter_actions/auxiliary_fmri property is set with a valid fmri,
1618          * copy the fmri to restarter/auxiliary_fmri so svcs -x can use.
1619          */
1620         if (reason == restarter_str_service_request &&
1621             libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &scf_inst) == 0) {
1622                 if (restarter_inst_validate_ractions_aux_fmri(scf_inst) == 0) {
1623                         if (restarter_inst_set_aux_fmri(scf_inst))
1624                                 log_framework(LOG_DEBUG, "%s: "
1625                                     "restarter_inst_set_aux_fmri failed: ",
1626                                     rip->ri_i.i_fmri);
1627                 } else {
1628                         log_framework(LOG_DEBUG, "%s: "
1629                             "restarter_inst_validate_ractions_aux_fmri "
1630                             "failed: ", rip->ri_i.i_fmri);
1631 
1632                         if (restarter_inst_reset_aux_fmri(scf_inst))
1633                                 log_framework(LOG_DEBUG, "%s: "
1634                                     "restarter_inst_reset_aux_fmri failed: ",
1635                                     rip->ri_i.i_fmri);
1636                 }
1637                 scf_instance_destroy(scf_inst);
1638         }
1639 
1640         if (immediate || !instance_started(rip)) {
1641                 if (rip->ri_i.i_primary_ctid != 0) {
1642                         rip->ri_m_inst = safe_scf_instance_create(h);
1643                         rip->ri_mi_deleted = B_FALSE;
1644 
1645                         libscf_reget_instance(rip);
1646                         method_remove_contract(rip, B_TRUE, B_TRUE);
1647 
1648                         scf_instance_destroy(rip->ri_m_inst);
1649                 }
1650 
1651                 (void) restarter_instance_update_states(h, rip,
1652                     RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, RERR_RESTART,
1653                     reason);
1654                 return;
1655         }
1656 
1657         (void) restarter_instance_update_states(h, rip, rip->ri_i.i_state,
1658             RESTARTER_STATE_MAINT, RERR_NONE, reason);
1659 
1660         log_transition(rip, MAINT_REQUESTED);
1661 
1662         info = startd_zalloc(sizeof (*info));
1663         info->sf_id = rip->ri_id;
1664         info->sf_method_type = METHOD_STOP;
1665         info->sf_event_type = RERR_RESTART;
1666         info->sf_reason = reason;
1667         rip->ri_method_thread = startd_thread_create(method_thread, info);
1668 }
1669 
1670 static void
1671 refresh_instance(scf_handle_t *h, restarter_inst_t *rip)
1672 {
1673         scf_instance_t *inst;
1674         scf_snapshot_t *snap;
1675         fork_info_t *info;
1676         int r;
1677 
1678         assert(MUTEX_HELD(&rip->ri_lock));
1679 
1680         log_instance(rip, B_TRUE, "Rereading configuration.");
1681         log_framework(LOG_DEBUG, "%s: rereading configuration.\n",
1682             rip->ri_i.i_fmri);
1683 
1684 rep_retry:
1685         r = libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst);
1686         switch (r) {
1687         case 0:
1688                 break;
1689 
1690         case ECONNABORTED:
1691                 libscf_handle_rebind(h);
1692                 goto rep_retry;
1693 
1694         case ENOENT:
1695                 /* Must have been deleted. */
1696                 return;
1697 
1698         case EINVAL:
1699         case ENOTSUP:
1700         default:
1701                 bad_error("libscf_fmri_get_instance", r);
1702         }
1703 
1704         snap = libscf_get_running_snapshot(inst);
1705 
1706         r = libscf_get_startd_properties(inst, snap, &rip->ri_flags,
1707             &rip->ri_utmpx_prefix);
1708         switch (r) {
1709         case 0:
1710                 log_framework(LOG_DEBUG, "%s is a %s-style service\n",
1711                     rip->ri_i.i_fmri, service_style(rip->ri_flags));
1712                 break;
1713 
1714         case ECONNABORTED:
1715                 scf_instance_destroy(inst);
1716                 scf_snapshot_destroy(snap);
1717                 libscf_handle_rebind(h);
1718                 goto rep_retry;
1719 
1720         case ECANCELED:
1721         case ENOENT:
1722                 /* Succeed in anticipation of REMOVE_INSTANCE. */
1723                 break;
1724 
1725         default:
1726                 bad_error("libscf_get_startd_properties", r);
1727         }
1728 
1729         if (instance_started(rip)) {
1730                 /* Refresh does not change the state. */
1731                 (void) restarter_instance_update_states(h, rip,
1732                     rip->ri_i.i_state, rip->ri_i.i_state, RERR_NONE,
1733                     restarter_str_refresh);
1734 
1735                 info = startd_zalloc(sizeof (*info));
1736                 info->sf_id = rip->ri_id;
1737                 info->sf_method_type = METHOD_REFRESH;
1738                 info->sf_event_type = RERR_REFRESH;
1739                 info->sf_reason = NULL;
1740 
1741                 assert(rip->ri_method_thread == 0);
1742                 rip->ri_method_thread =
1743                     startd_thread_create(method_thread, info);
1744         }
1745 
1746         scf_snapshot_destroy(snap);
1747         scf_instance_destroy(inst);
1748 }
1749 
1750 const char *event_names[] = { "INVALID", "ADD_INSTANCE", "REMOVE_INSTANCE",
1751         "ENABLE", "DISABLE", "ADMIN_DEGRADED", "ADMIN_REFRESH",
1752         "ADMIN_RESTART", "ADMIN_MAINT_OFF", "ADMIN_MAINT_ON",
1753         "ADMIN_MAINT_ON_IMMEDIATE", "STOP", "START", "DEPENDENCY_CYCLE",
1754         "INVALID_DEPENDENCY", "ADMIN_DISABLE", "STOP_RESET"
1755 };
1756 
1757 /*
1758  * void *restarter_process_events()
1759  *
1760  *   Called in a separate thread to process the events on an instance's
1761  *   queue.  Empties the queue completely, and tries to keep the thread
1762  *   around for a little while after the queue is empty to save on
1763  *   startup costs.
1764  */
1765 static void *
1766 restarter_process_events(void *arg)
1767 {
1768         scf_handle_t *h;
1769         restarter_instance_qentry_t *event;
1770         restarter_inst_t *rip;
1771         char *fmri = (char *)arg;
1772         struct timespec to;
1773 
1774         assert(fmri != NULL);
1775 
1776         h = libscf_handle_create_bound_loop();
1777 
1778         /* grab the queue lock */
1779         rip = inst_lookup_queue(fmri);
1780         if (rip == NULL)
1781                 goto out;
1782 
1783 again:
1784 
1785         while ((event = uu_list_first(rip->ri_queue)) != NULL) {
1786                 restarter_inst_t *inst;
1787 
1788                 /* drop the queue lock */
1789                 MUTEX_UNLOCK(&rip->ri_queue_lock);
1790 
1791                 /*
1792                  * Grab the inst lock -- this waits until any outstanding
1793                  * method finishes running.
1794                  */
1795                 inst = inst_lookup_by_name(fmri);
1796                 if (inst == NULL) {
1797                         /* Getting deleted in the middle isn't an error. */
1798                         goto cont;
1799                 }
1800 
1801                 assert(instance_in_transition(inst) == 0);
1802 
1803                 /* process the event */
1804                 switch (event->riq_type) {
1805                 case RESTARTER_EVENT_TYPE_ENABLE:
1806                 case RESTARTER_EVENT_TYPE_DISABLE:
1807                         (void) enable_inst(h, inst, event);
1808                         break;
1809 
1810                 case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
1811                         if (enable_inst(h, inst, event) == 0)
1812                                 reset_start_times(inst);
1813                         break;
1814 
1815                 case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
1816                         restarter_delete_inst(inst);
1817                         inst = NULL;
1818                         goto cont;
1819 
1820                 case RESTARTER_EVENT_TYPE_STOP_RESET:
1821                         reset_start_times(inst);
1822                         /* FALLTHROUGH */
1823                 case RESTARTER_EVENT_TYPE_STOP:
1824                         (void) stop_instance(h, inst, RSTOP_DEPENDENCY);
1825                         break;
1826 
1827                 case RESTARTER_EVENT_TYPE_START:
1828                         start_instance(h, inst, event->riq_reason);
1829                         break;
1830 
1831                 case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
1832                         maintain_instance(h, inst, 0,
1833                             restarter_str_dependency_cycle);
1834                         break;
1835 
1836                 case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
1837                         maintain_instance(h, inst, 0,
1838                             restarter_str_invalid_dependency);
1839                         break;
1840 
1841                 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
1842                         if (event_from_tty(h, inst) == 0)
1843                                 maintain_instance(h, inst, 0,
1844                                     restarter_str_service_request);
1845                         else
1846                                 maintain_instance(h, inst, 0,
1847                                     restarter_str_administrative_request);
1848                         break;
1849 
1850                 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
1851                         if (event_from_tty(h, inst) == 0)
1852                                 maintain_instance(h, inst, 1,
1853                                     restarter_str_service_request);
1854                         else
1855                                 maintain_instance(h, inst, 1,
1856                                     restarter_str_administrative_request);
1857                         break;
1858 
1859                 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
1860                         unmaintain_instance(h, inst, RUNMAINT_CLEAR);
1861                         reset_start_times(inst);
1862                         break;
1863 
1864                 case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
1865                         refresh_instance(h, inst);
1866                         break;
1867 
1868                 case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
1869                         log_framework(LOG_WARNING, "Restarter: "
1870                             "%s command (for %s) unimplemented.\n",
1871                             event_names[event->riq_type], inst->ri_i.i_fmri);
1872                         break;
1873 
1874                 case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
1875                         if (!instance_started(inst)) {
1876                                 log_framework(LOG_DEBUG, "Restarter: "
1877                                     "Not restarting %s; not running.\n",
1878                                     inst->ri_i.i_fmri);
1879                         } else {
1880                                 /*
1881                                  * Stop the instance.  If it can be restarted,
1882                                  * the graph engine will send a new event.
1883                                  */
1884                                 if (restart_dump(h, inst)) {
1885                                         (void) contract_kill(
1886                                             inst->ri_i.i_primary_ctid, SIGABRT,
1887                                             inst->ri_i.i_fmri);
1888                                 } else if (stop_instance(h, inst,
1889                                     RSTOP_RESTART) == 0) {
1890                                         reset_start_times(inst);
1891                                 }
1892                         }
1893                         break;
1894 
1895                 case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
1896                 default:
1897 #ifndef NDEBUG
1898                         uu_warn("%s:%d: Bad restarter event %d.  "
1899                             "Aborting.\n", __FILE__, __LINE__, event->riq_type);
1900 #endif
1901                         abort();
1902                 }
1903 
1904                 assert(inst != NULL);
1905                 MUTEX_UNLOCK(&inst->ri_lock);
1906 
1907 cont:
1908                 /* grab the queue lock */
1909                 rip = inst_lookup_queue(fmri);
1910                 if (rip == NULL)
1911                         goto out;
1912 
1913                 /* delete the event */
1914                 uu_list_remove(rip->ri_queue, event);
1915                 startd_free(event, sizeof (restarter_instance_qentry_t));
1916         }
1917 
1918         assert(rip != NULL);
1919 
1920         /*
1921          * Try to preserve the thread for a little while for future use.
1922          */
1923         to.tv_sec = 3;
1924         to.tv_nsec = 0;
1925         (void) pthread_cond_reltimedwait_np(&rip->ri_queue_cv,
1926             &rip->ri_queue_lock, &to);
1927 
1928         if (uu_list_first(rip->ri_queue) != NULL)
1929                 goto again;
1930 
1931         rip->ri_queue_thread = 0;
1932         MUTEX_UNLOCK(&rip->ri_queue_lock);
1933 
1934 out:
1935         (void) scf_handle_unbind(h);
1936         scf_handle_destroy(h);
1937         free(fmri);
1938         return (NULL);
1939 }
1940 
1941 static int
1942 is_admin_event(restarter_event_type_t t) {
1943 
1944         switch (t) {
1945         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
1946         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
1947         case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
1948         case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
1949         case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
1950         case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
1951                 return (1);
1952         default:
1953                 return (0);
1954         }
1955 }
1956 
1957 static void
1958 restarter_queue_event(restarter_inst_t *ri, restarter_protocol_event_t *e)
1959 {
1960         restarter_instance_qentry_t *qe;
1961         int r;
1962 
1963         assert(MUTEX_HELD(&ri->ri_queue_lock));
1964         assert(!MUTEX_HELD(&ri->ri_lock));
1965 
1966         qe = startd_zalloc(sizeof (restarter_instance_qentry_t));
1967         qe->riq_type = e->rpe_type;
1968         qe->riq_reason = e->rpe_reason;
1969 
1970         uu_list_node_init(qe, &qe->riq_link, restarter_queue_pool);
1971         r = uu_list_insert_before(ri->ri_queue, NULL, qe);
1972         assert(r == 0);
1973 }
1974 
1975 /*
1976  * void *restarter_event_thread()
1977  *
1978  *  Handle incoming graph events by placing them on a per-instance
1979  *  queue.  We can't lock the main part of the instance structure, so
1980  *  just modify the seprarately locked event queue portion.
1981  */
1982 /*ARGSUSED*/
1983 static void *
1984 restarter_event_thread(void *unused)
1985 {
1986         scf_handle_t *h;
1987 
1988         /*
1989          * This is a new thread, and thus, gets its own handle
1990          * to the repository.
1991          */
1992         h = libscf_handle_create_bound_loop();
1993 
1994         MUTEX_LOCK(&ru->restarter_update_lock);
1995 
1996         /*CONSTCOND*/
1997         while (1) {
1998                 restarter_protocol_event_t *e;
1999 
2000                 while (ru->restarter_update_wakeup == 0)
2001                         (void) pthread_cond_wait(&ru->restarter_update_cv,
2002                             &ru->restarter_update_lock);
2003 
2004                 ru->restarter_update_wakeup = 0;
2005 
2006                 while ((e = restarter_event_dequeue()) != NULL) {
2007                         restarter_inst_t *rip;
2008                         char *fmri;
2009 
2010                         MUTEX_UNLOCK(&ru->restarter_update_lock);
2011 
2012                         /*
2013                          * ADD_INSTANCE is special: there's likely no
2014                          * instance structure yet, so we need to handle the
2015                          * addition synchronously.
2016                          */
2017                         switch (e->rpe_type) {
2018                         case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
2019                                 if (restarter_insert_inst(h, e->rpe_inst) != 0)
2020                                         log_error(LOG_INFO, "Restarter: "
2021                                             "Could not add %s.\n", e->rpe_inst);
2022 
2023                                 MUTEX_LOCK(&st->st_load_lock);
2024                                 if (--st->st_load_instances == 0)
2025                                         (void) pthread_cond_broadcast(
2026                                             &st->st_load_cv);
2027                                 MUTEX_UNLOCK(&st->st_load_lock);
2028 
2029                                 goto nolookup;
2030                         }
2031 
2032                         /*
2033                          * Lookup the instance, locking only the event queue.
2034                          * Can't grab ri_lock here because it might be held
2035                          * by a long-running method.
2036                          */
2037                         rip = inst_lookup_queue(e->rpe_inst);
2038                         if (rip == NULL) {
2039                                 log_error(LOG_INFO, "Restarter: "
2040                                     "Ignoring %s command for unknown service "
2041                                     "%s.\n", event_names[e->rpe_type],
2042                                     e->rpe_inst);
2043                                 goto nolookup;
2044                         }
2045 
2046                         /* Keep ADMIN events from filling up the queue. */
2047                         if (is_admin_event(e->rpe_type) &&
2048                             uu_list_numnodes(rip->ri_queue) >
2049                             RINST_QUEUE_THRESHOLD) {
2050                                 MUTEX_UNLOCK(&rip->ri_queue_lock);
2051                                 log_instance(rip, B_TRUE, "Instance event "
2052                                     "queue overflow.  Dropping administrative "
2053                                     "request.");
2054                                 log_framework(LOG_DEBUG, "%s: Instance event "
2055                                     "queue overflow.  Dropping administrative "
2056                                     "request.\n", rip->ri_i.i_fmri);
2057                                 goto nolookup;
2058                         }
2059 
2060                         /* Now add the event to the instance queue. */
2061                         restarter_queue_event(rip, e);
2062 
2063                         if (rip->ri_queue_thread == 0) {
2064                                 /*
2065                                  * Start a thread if one isn't already
2066                                  * running.
2067                                  */
2068                                 fmri = safe_strdup(e->rpe_inst);
2069                                 rip->ri_queue_thread =  startd_thread_create(
2070                                     restarter_process_events, (void *)fmri);
2071                         } else {
2072                                 /*
2073                                  * Signal the existing thread that there's
2074                                  * a new event.
2075                                  */
2076                                 (void) pthread_cond_broadcast(
2077                                     &rip->ri_queue_cv);
2078                         }
2079 
2080                         MUTEX_UNLOCK(&rip->ri_queue_lock);
2081 nolookup:
2082                         restarter_event_release(e);
2083 
2084                         MUTEX_LOCK(&ru->restarter_update_lock);
2085                 }
2086         }
2087 
2088         /*
2089          * Unreachable for now -- there's currently no graceful cleanup
2090          * called on exit().
2091          */
2092         (void) scf_handle_unbind(h);
2093         scf_handle_destroy(h);
2094         return (NULL);
2095 }
2096 
2097 static restarter_inst_t *
2098 contract_to_inst(ctid_t ctid)
2099 {
2100         restarter_inst_t *inst;
2101         int id;
2102 
2103         id = lookup_inst_by_contract(ctid);
2104         if (id == -1)
2105                 return (NULL);
2106 
2107         inst = inst_lookup_by_id(id);
2108         if (inst != NULL) {
2109                 /*
2110                  * Since ri_lock isn't held by the contract id lookup, this
2111                  * instance may have been restarted and now be in a new
2112                  * contract, making the old contract no longer valid for this
2113                  * instance.
2114                  */
2115                 if (ctid != inst->ri_i.i_primary_ctid) {
2116                         MUTEX_UNLOCK(&inst->ri_lock);
2117                         inst = NULL;
2118                 }
2119         }
2120         return (inst);
2121 }
2122 
2123 /*
2124  * void contract_action()
2125  *   Take action on contract events.
2126  */
2127 static void
2128 contract_action(scf_handle_t *h, restarter_inst_t *inst, ctid_t id,
2129     uint32_t type)
2130 {
2131         const char *fmri = inst->ri_i.i_fmri;
2132 
2133         assert(MUTEX_HELD(&inst->ri_lock));
2134 
2135         /*
2136          * If startd has stopped this contract, there is no need to
2137          * stop it again.
2138          */
2139         if (inst->ri_i.i_primary_ctid > 0 &&
2140             inst->ri_i.i_primary_ctid_stopped)
2141                 return;
2142 
2143         if ((type & (CT_PR_EV_EMPTY | CT_PR_EV_CORE | CT_PR_EV_SIGNAL
2144             | CT_PR_EV_HWERR)) == 0) {
2145                 /*
2146                  * There shouldn't be other events, since that's not how we set
2147                  * the terms. Thus, just log an error and drive on.
2148                  */
2149                 log_framework(LOG_NOTICE,
2150                     "%s: contract %ld received unexpected critical event "
2151                     "(%d)\n", fmri, id, type);
2152                 return;
2153         }
2154 
2155         assert(instance_in_transition(inst) == 0);
2156 
2157         if (instance_is_wait_style(inst)) {
2158                 /*
2159                  * We ignore all events; if they impact the
2160                  * process we're monitoring, then the
2161                  * wait_thread will stop the instance.
2162                  */
2163                 log_framework(LOG_DEBUG,
2164                     "%s: ignoring contract event on wait-style service\n",
2165                     fmri);
2166         } else {
2167                 /*
2168                  * A CT_PR_EV_EMPTY event is an RSTOP_EXIT request.
2169                  */
2170                 switch (type) {
2171                 case CT_PR_EV_EMPTY:
2172                         (void) stop_instance(h, inst, RSTOP_EXIT);
2173                         break;
2174                 case CT_PR_EV_CORE:
2175                         (void) stop_instance(h, inst, RSTOP_CORE);
2176                         break;
2177                 case CT_PR_EV_SIGNAL:
2178                         (void) stop_instance(h, inst, RSTOP_SIGNAL);
2179                         break;
2180                 case CT_PR_EV_HWERR:
2181                         (void) stop_instance(h, inst, RSTOP_HWERR);
2182                         break;
2183                 }
2184         }
2185 }
2186 
2187 /*
2188  * void *restarter_contract_event_thread(void *)
2189  *   Listens to the process contract bundle for critical events, taking action
2190  *   on events from contracts we know we are responsible for.
2191  */
2192 /*ARGSUSED*/
2193 static void *
2194 restarter_contracts_event_thread(void *unused)
2195 {
2196         int fd, err;
2197         scf_handle_t *local_handle;
2198 
2199         /*
2200          * Await graph load completion.  That is, stop here, until we've scanned
2201          * the repository for contract - instance associations.
2202          */
2203         MUTEX_LOCK(&st->st_load_lock);
2204         while (!(st->st_load_complete && st->st_load_instances == 0))
2205                 (void) pthread_cond_wait(&st->st_load_cv, &st->st_load_lock);
2206         MUTEX_UNLOCK(&st->st_load_lock);
2207 
2208         /*
2209          * This is a new thread, and thus, gets its own handle
2210          * to the repository.
2211          */
2212         if ((local_handle = libscf_handle_create_bound(SCF_VERSION)) == NULL)
2213                 uu_die("Unable to bind a new repository handle: %s\n",
2214                     scf_strerror(scf_error()));
2215 
2216         fd = open64(CTFS_ROOT "/process/pbundle", O_RDONLY);
2217         if (fd == -1)
2218                 uu_die("process bundle open failed");
2219 
2220         /*
2221          * Make sure we get all events (including those generated by configd
2222          * before this thread was started).
2223          */
2224         err = ct_event_reset(fd);
2225         assert(err == 0);
2226 
2227         for (;;) {
2228                 int efd, sfd;
2229                 ct_evthdl_t ev;
2230                 uint32_t type;
2231                 ctevid_t evid;
2232                 ct_stathdl_t status;
2233                 ctid_t ctid;
2234                 restarter_inst_t *inst;
2235                 uint64_t cookie;
2236 
2237                 if (err = ct_event_read_critical(fd, &ev)) {
2238                         log_error(LOG_WARNING,
2239                             "Error reading next contract event: %s",
2240                             strerror(err));
2241                         continue;
2242                 }
2243 
2244                 evid = ct_event_get_evid(ev);
2245                 ctid = ct_event_get_ctid(ev);
2246                 type = ct_event_get_type(ev);
2247 
2248                 /* Fetch cookie. */
2249                 if ((sfd = contract_open(ctid, "process", "status", O_RDONLY))
2250                     < 0) {
2251                         ct_event_free(ev);
2252                         continue;
2253                 }
2254 
2255                 if (err = ct_status_read(sfd, CTD_COMMON, &status)) {
2256                         log_framework(LOG_WARNING, "Could not get status for "
2257                             "contract %ld: %s\n", ctid, strerror(err));
2258 
2259                         startd_close(sfd);
2260                         ct_event_free(ev);
2261                         continue;
2262                 }
2263 
2264                 cookie = ct_status_get_cookie(status);
2265 
2266                 log_framework(LOG_DEBUG, "Received event %d for ctid %ld "
2267                     "cookie %lld\n", type, ctid, cookie);
2268 
2269                 ct_status_free(status);
2270 
2271                 startd_close(sfd);
2272 
2273                 /*
2274                  * svc.configd(1M) restart handling performed by the
2275                  * fork_configd_thread.  We don't acknowledge, as that thread
2276                  * will do so.
2277                  */
2278                 if (cookie == CONFIGD_COOKIE) {
2279                         ct_event_free(ev);
2280                         continue;
2281                 }
2282 
2283                 inst = NULL;
2284                 if (storing_contract != 0 &&
2285                     (inst = contract_to_inst(ctid)) == NULL) {
2286                         /*
2287                          * This can happen for two reasons:
2288                          * - method_run() has not yet stored the
2289                          *    the contract into the internal hash table.
2290                          * - we receive an EMPTY event for an abandoned
2291                          *    contract.
2292                          * If there is any contract in the process of
2293                          * being stored into the hash table then re-read
2294                          * the event later.
2295                          */
2296                         log_framework(LOG_DEBUG,
2297                             "Reset event %d for unknown "
2298                             "contract id %ld\n", type, ctid);
2299 
2300                         /* don't go too fast */
2301                         (void) poll(NULL, 0, 100);
2302 
2303                         (void) ct_event_reset(fd);
2304                         ct_event_free(ev);
2305                         continue;
2306                 }
2307 
2308                 /*
2309                  * Do not call contract_to_inst() again if first
2310                  * call succeeded.
2311                  */
2312                 if (inst == NULL)
2313                         inst = contract_to_inst(ctid);
2314                 if (inst == NULL) {
2315                         /*
2316                          * This can happen if we receive an EMPTY
2317                          * event for an abandoned contract.
2318                          */
2319                         log_framework(LOG_DEBUG,
2320                             "Received event %d for unknown contract id "
2321                             "%ld\n", type, ctid);
2322                 } else {
2323                         log_framework(LOG_DEBUG,
2324                             "Received event %d for contract id "
2325                             "%ld (%s)\n", type, ctid,
2326                             inst->ri_i.i_fmri);
2327 
2328                         contract_action(local_handle, inst, ctid, type);
2329 
2330                         MUTEX_UNLOCK(&inst->ri_lock);
2331                 }
2332 
2333                 efd = contract_open(ct_event_get_ctid(ev), "process", "ctl",
2334                     O_WRONLY);
2335                 if (efd != -1) {
2336                         (void) ct_ctl_ack(efd, evid);
2337                         startd_close(efd);
2338                 }
2339 
2340                 ct_event_free(ev);
2341 
2342         }
2343 
2344         /*NOTREACHED*/
2345         return (NULL);
2346 }
2347 
2348 /*
2349  * Timeout queue, processed by restarter_timeouts_event_thread().
2350  */
2351 timeout_queue_t *timeouts;
2352 static uu_list_pool_t *timeout_pool;
2353 
2354 typedef struct timeout_update {
2355         pthread_mutex_t         tu_lock;
2356         pthread_cond_t          tu_cv;
2357         int                     tu_wakeup;
2358 } timeout_update_t;
2359 
2360 timeout_update_t *tu;
2361 
2362 static const char *timeout_ovr_svcs[] = {
2363         "svc:/system/manifest-import:default",
2364         "svc:/network/initial:default",
2365         "svc:/network/service:default",
2366         "svc:/system/rmtmpfiles:default",
2367         "svc:/network/loopback:default",
2368         "svc:/network/physical:default",
2369         "svc:/system/device/local:default",
2370         "svc:/system/filesystem/usr:default",
2371         "svc:/system/filesystem/minimal:default",
2372         "svc:/system/filesystem/local:default",
2373         NULL
2374 };
2375 
2376 int
2377 is_timeout_ovr(restarter_inst_t *inst)
2378 {
2379         int i;
2380 
2381         for (i = 0; timeout_ovr_svcs[i] != NULL; ++i) {
2382                 if (strcmp(inst->ri_i.i_fmri, timeout_ovr_svcs[i]) == 0) {
2383                         log_instance(inst, B_TRUE, "Timeout override by "
2384                             "svc.startd.  Using infinite timeout.");
2385                         return (1);
2386                 }
2387         }
2388 
2389         return (0);
2390 }
2391 
2392 /*ARGSUSED*/
2393 static int
2394 timeout_compare(const void *lc_arg, const void *rc_arg, void *private)
2395 {
2396         hrtime_t t1 = ((const timeout_entry_t *)lc_arg)->te_timeout;
2397         hrtime_t t2 = ((const timeout_entry_t *)rc_arg)->te_timeout;
2398 
2399         if (t1 > t2)
2400                 return (1);
2401         else if (t1 < t2)
2402                 return (-1);
2403         return (0);
2404 }
2405 
2406 void
2407 timeout_init()
2408 {
2409         timeouts = startd_zalloc(sizeof (timeout_queue_t));
2410 
2411         (void) pthread_mutex_init(&timeouts->tq_lock, &mutex_attrs);
2412 
2413         timeout_pool = startd_list_pool_create("timeouts",
2414             sizeof (timeout_entry_t), offsetof(timeout_entry_t, te_link),
2415             timeout_compare, UU_LIST_POOL_DEBUG);
2416         assert(timeout_pool != NULL);
2417 
2418         timeouts->tq_list = startd_list_create(timeout_pool,
2419             timeouts, UU_LIST_SORTED);
2420         assert(timeouts->tq_list != NULL);
2421 
2422         tu = startd_zalloc(sizeof (timeout_update_t));
2423         (void) pthread_cond_init(&tu->tu_cv, NULL);
2424         (void) pthread_mutex_init(&tu->tu_lock, &mutex_attrs);
2425 }
2426 
2427 void
2428 timeout_insert(restarter_inst_t *inst, ctid_t cid, uint64_t timeout_sec)
2429 {
2430         hrtime_t now, timeout;
2431         timeout_entry_t *entry;
2432         uu_list_index_t idx;
2433 
2434         assert(MUTEX_HELD(&inst->ri_lock));
2435 
2436         now = gethrtime();
2437 
2438         /*
2439          * If we overflow LLONG_MAX, we're never timing out anyways, so
2440          * just return.
2441          */
2442         if (timeout_sec >= (LLONG_MAX - now) / 1000000000LL) {
2443                 log_instance(inst, B_TRUE, "timeout_seconds too large, "
2444                     "treating as infinite.");
2445                 return;
2446         }
2447 
2448         /* hrtime is in nanoseconds. Convert timeout_sec. */
2449         timeout = now + (timeout_sec * 1000000000LL);
2450 
2451         entry = startd_alloc(sizeof (timeout_entry_t));
2452         entry->te_timeout = timeout;
2453         entry->te_ctid = cid;
2454         entry->te_fmri = safe_strdup(inst->ri_i.i_fmri);
2455         entry->te_logstem = safe_strdup(inst->ri_logstem);
2456         entry->te_fired = 0;
2457         /* Insert the calculated timeout time onto the queue. */
2458         MUTEX_LOCK(&timeouts->tq_lock);
2459         (void) uu_list_find(timeouts->tq_list, entry, NULL, &idx);
2460         uu_list_node_init(entry, &entry->te_link, timeout_pool);
2461         uu_list_insert(timeouts->tq_list, entry, idx);
2462         MUTEX_UNLOCK(&timeouts->tq_lock);
2463 
2464         assert(inst->ri_timeout == NULL);
2465         inst->ri_timeout = entry;
2466 
2467         MUTEX_LOCK(&tu->tu_lock);
2468         tu->tu_wakeup = 1;
2469         (void) pthread_cond_broadcast(&tu->tu_cv);
2470         MUTEX_UNLOCK(&tu->tu_lock);
2471 }
2472 
2473 
2474 void
2475 timeout_remove(restarter_inst_t *inst, ctid_t cid)
2476 {
2477         assert(MUTEX_HELD(&inst->ri_lock));
2478 
2479         if (inst->ri_timeout == NULL)
2480                 return;
2481 
2482         assert(inst->ri_timeout->te_ctid == cid);
2483 
2484         MUTEX_LOCK(&timeouts->tq_lock);
2485         uu_list_remove(timeouts->tq_list, inst->ri_timeout);
2486         MUTEX_UNLOCK(&timeouts->tq_lock);
2487 
2488         free(inst->ri_timeout->te_fmri);
2489         free(inst->ri_timeout->te_logstem);
2490         startd_free(inst->ri_timeout, sizeof (timeout_entry_t));
2491         inst->ri_timeout = NULL;
2492 }
2493 
2494 static int
2495 timeout_now()
2496 {
2497         timeout_entry_t *e;
2498         hrtime_t now;
2499         int ret;
2500 
2501         now = gethrtime();
2502 
2503         /*
2504          * Walk through the (sorted) timeouts list.  While the timeout
2505          * at the head of the list is <= the current time, kill the
2506          * method.
2507          */
2508         MUTEX_LOCK(&timeouts->tq_lock);
2509 
2510         for (e = uu_list_first(timeouts->tq_list);
2511             e != NULL && e->te_timeout <= now;
2512             e = uu_list_next(timeouts->tq_list, e)) {
2513                 log_framework(LOG_WARNING, "%s: Method or service exit timed "
2514                     "out.  Killing contract %ld.\n", e->te_fmri, e->te_ctid);
2515                 log_instance_fmri(e->te_fmri, e->te_logstem, B_TRUE,
2516                     "Method or service exit timed out.  Killing contract %ld.",
2517                     e->te_ctid);
2518                 e->te_fired = 1;
2519                 (void) contract_kill(e->te_ctid, SIGKILL, e->te_fmri);
2520         }
2521 
2522         if (uu_list_numnodes(timeouts->tq_list) > 0)
2523                 ret = 0;
2524         else
2525                 ret = -1;
2526 
2527         MUTEX_UNLOCK(&timeouts->tq_lock);
2528 
2529         return (ret);
2530 }
2531 
2532 /*
2533  * void *restarter_timeouts_event_thread(void *)
2534  *   Responsible for monitoring the method timeouts.  This thread must
2535  *   be started before any methods are called.
2536  */
2537 /*ARGSUSED*/
2538 static void *
2539 restarter_timeouts_event_thread(void *unused)
2540 {
2541         /*
2542          * Timeouts are entered on a priority queue, which is processed by
2543          * this thread.  As timeouts are specified in seconds, we'll do
2544          * the necessary processing every second, as long as the queue
2545          * is not empty.
2546          */
2547 
2548         /*CONSTCOND*/
2549         while (1) {
2550                 /*
2551                  * As long as the timeout list isn't empty, process it
2552                  * every second.
2553                  */
2554                 if (timeout_now() == 0) {
2555                         (void) sleep(1);
2556                         continue;
2557                 }
2558 
2559                 /* The list is empty, wait until we have more timeouts. */
2560                 MUTEX_LOCK(&tu->tu_lock);
2561 
2562                 while (tu->tu_wakeup == 0)
2563                         (void) pthread_cond_wait(&tu->tu_cv, &tu->tu_lock);
2564 
2565                 tu->tu_wakeup = 0;
2566                 MUTEX_UNLOCK(&tu->tu_lock);
2567         }
2568 
2569         return (NULL);
2570 }
2571 
2572 void
2573 restarter_start()
2574 {
2575         (void) startd_thread_create(restarter_timeouts_event_thread, NULL);
2576         (void) startd_thread_create(restarter_event_thread, NULL);
2577         (void) startd_thread_create(restarter_contracts_event_thread, NULL);
2578         (void) startd_thread_create(wait_thread, NULL);
2579 }
2580 
2581 
2582 void
2583 restarter_init()
2584 {
2585         restarter_instance_pool = startd_list_pool_create("restarter_instances",
2586             sizeof (restarter_inst_t), offsetof(restarter_inst_t,
2587             ri_link), restarter_instance_compare, UU_LIST_POOL_DEBUG);
2588         (void) memset(&instance_list, 0, sizeof (instance_list));
2589 
2590         (void) pthread_mutex_init(&instance_list.ril_lock, &mutex_attrs);
2591         instance_list.ril_instance_list = startd_list_create(
2592             restarter_instance_pool, &instance_list, UU_LIST_SORTED);
2593 
2594         restarter_queue_pool = startd_list_pool_create(
2595             "restarter_instance_queue", sizeof (restarter_instance_qentry_t),
2596             offsetof(restarter_instance_qentry_t,  riq_link), NULL,
2597             UU_LIST_POOL_DEBUG);
2598 
2599         contract_list_pool = startd_list_pool_create(
2600             "contract_list", sizeof (contract_entry_t),
2601             offsetof(contract_entry_t,  ce_link), NULL,
2602             UU_LIST_POOL_DEBUG);
2603         contract_hash_init();
2604 
2605         log_framework(LOG_DEBUG, "Initialized restarter\n");
2606 }