1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2011 Joyent Inc.
  25  */
  26 
  27 /*
  28  * method.c - method execution functions
  29  *
  30  * This file contains the routines needed to run a method:  a fork(2)-exec(2)
  31  * invocation monitored using either the contract filesystem or waitpid(2).
  32  * (Plain fork1(2) support is provided in fork.c.)
  33  *
  34  * Contract Transfer
  35  *   When we restart a service, we want to transfer any contracts that the old
  36  *   service's contract inherited.  This means that (a) we must not abandon the
  37  *   old contract when the service dies and (b) we must write the id of the old
  38  *   contract into the terms of the new contract.  There should be limits to
  39  *   (a), though, since we don't want to keep the contract around forever.  To
  40  *   this end we'll say that services in the offline state may have a contract
  41  *   to be transfered and services in the disabled or maintenance states cannot.
  42  *   This means that when a service transitions from online (or degraded) to
  43  *   offline, the contract should be preserved, and when the service transitions
  44  *   from offline to online (i.e., the start method), we'll transfer inherited
  45  *   contracts.
  46  */
  47 
  48 #include <sys/contract/process.h>
  49 #include <sys/ctfs.h>
  50 #include <sys/stat.h>
  51 #include <sys/time.h>
  52 #include <sys/types.h>
  53 #include <sys/uio.h>
  54 #include <sys/wait.h>
  55 #include <alloca.h>
  56 #include <assert.h>
  57 #include <errno.h>
  58 #include <fcntl.h>
  59 #include <libcontract.h>
  60 #include <libcontract_priv.h>
  61 #include <libgen.h>
  62 #include <librestart.h>
  63 #include <libscf.h>
  64 #include <limits.h>
  65 #include <port.h>
  66 #include <sac.h>
  67 #include <signal.h>
  68 #include <stdlib.h>
  69 #include <string.h>
  70 #include <strings.h>
  71 #include <unistd.h>
  72 #include <atomic.h>
  73 #include <poll.h>
  74 #include <libscf_priv.h>
  75 
  76 #include "startd.h"
  77 
  78 #define SBIN_SH         "/sbin/sh"
  79 
  80 /*
  81  * Used to tell if contracts are in the process of being
  82  * stored into the svc.startd internal hash table.
  83  */
  84 volatile uint16_t       storing_contract = 0;
  85 
  86 /*
  87  * Mapping from restart_on method-type to contract events.  Must correspond to
  88  * enum method_restart_t.
  89  */
  90 static uint_t method_events[] = {
  91         /* METHOD_RESTART_ALL */
  92         CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE | CT_PR_EV_EMPTY,
  93         /* METHOD_RESTART_EXTERNAL_FAULT */
  94         CT_PR_EV_HWERR | CT_PR_EV_SIGNAL,
  95         /* METHOD_RESTART_ANY_FAULT */
  96         CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE
  97 };
  98 
  99 /*
 100  * method_record_start(restarter_inst_t *)
 101  *   Record a service start for rate limiting.  Place the current time
 102  *   in the circular array of instance starts.
 103  */
 104 static void
 105 method_record_start(restarter_inst_t *inst)
 106 {
 107         int index = inst->ri_start_index++ % RINST_START_TIMES;
 108 
 109         inst->ri_start_time[index] = gethrtime();
 110 }
 111 
 112 /*
 113  * method_rate_critical(restarter_inst_t *)
 114  *    Return true if the average start interval is less than the permitted
 115  *    interval.  The implicit interval defaults to RINST_FAILURE_RATE_NS and
 116  *    RINST_START_TIMES but may be overridden with the svc properties
 117  *    startd/critical_failure_count and startd/critical_failure_period
 118  *    which represent the number of failures to consider and the amount of
 119  *    time in seconds in which that number may occur, respectively. Note that
 120  *    this time is measured as of the transition to 'enabled' rather than wall
 121  *    clock time.
 122  *    Implicit success if insufficient measurements for an average exist.
 123  */
 124 static int
 125 method_rate_critical(restarter_inst_t *inst)
 126 {
 127         hrtime_t critical_failure_period = RINST_FAILURE_RATE_NS;
 128         uint_t critical_failure_count = RINST_START_TIMES;
 129         uint_t n = inst->ri_start_index;
 130         hrtime_t avg_ns = 0;
 131         uint64_t scf_fr, scf_st;
 132         scf_propvec_t *prop = NULL;
 133         scf_propvec_t restart_critical[] = {
 134                 { "critical_failure_period", NULL, SCF_TYPE_INTEGER, NULL, 0 },
 135                 { "critical_failure_count", NULL, SCF_TYPE_INTEGER, NULL, 0 },
 136                 { NULL }
 137         };
 138 
 139         restart_critical[0].pv_ptr = &scf_fr;
 140         restart_critical[1].pv_ptr = &scf_st;
 141 
 142         if (scf_read_propvec(inst->ri_i.i_fmri, "startd",
 143             B_TRUE, restart_critical, &prop) != SCF_FAILED) {
 144                 /*
 145                  * critical_failure_period is expressed
 146                  * in seconds but tracked in ns
 147                  */
 148                 critical_failure_period = (hrtime_t)scf_fr * NANOSEC;
 149                 critical_failure_count = (uint_t)scf_st;
 150         }
 151         if (inst->ri_start_index < critical_failure_count)
 152                 return (0);
 153 
 154         avg_ns =
 155             (inst->ri_start_time[(n - 1) % critical_failure_count] -
 156             inst->ri_start_time[n % critical_failure_count]) /
 157             (critical_failure_count - 1);
 158 
 159         return (avg_ns < critical_failure_period);
 160 }
 161 
 162 /*
 163  * int method_is_transient()
 164  *   Determine if the method for the given instance is transient,
 165  *   from a contract perspective. Return 1 if it is, and 0 if it isn't.
 166  */
 167 static int
 168 method_is_transient(restarter_inst_t *inst, int type)
 169 {
 170         if (instance_is_transient_style(inst) || type != METHOD_START)
 171                 return (1);
 172         else
 173                 return (0);
 174 }
 175 
 176 /*
 177  * void method_store_contract()
 178  *   Store the newly created contract id into local structures and
 179  *   the repository.  If the repository connection is broken it is rebound.
 180  */
 181 static void
 182 method_store_contract(restarter_inst_t *inst, int type, ctid_t *cid)
 183 {
 184         int r;
 185         boolean_t primary;
 186 
 187         if (errno = contract_latest(cid))
 188                 uu_die("%s: Couldn't get new contract's id", inst->ri_i.i_fmri);
 189 
 190         primary = !method_is_transient(inst, type);
 191 
 192         if (!primary) {
 193                 if (inst->ri_i.i_transient_ctid != 0) {
 194                         log_framework(LOG_INFO,
 195                             "%s: transient ctid expected to be 0 but "
 196                             "was set to %ld\n", inst->ri_i.i_fmri,
 197                             inst->ri_i.i_transient_ctid);
 198                 }
 199 
 200                 inst->ri_i.i_transient_ctid = *cid;
 201         } else {
 202                 if (inst->ri_i.i_primary_ctid != 0) {
 203                         /*
 204                          * There was an old contract that we transferred.
 205                          * Remove it.
 206                          */
 207                         method_remove_contract(inst, B_TRUE, B_FALSE);
 208                 }
 209 
 210                 if (inst->ri_i.i_primary_ctid != 0) {
 211                         log_framework(LOG_INFO,
 212                             "%s: primary ctid expected to be 0 but "
 213                             "was set to %ld\n", inst->ri_i.i_fmri,
 214                             inst->ri_i.i_primary_ctid);
 215                 }
 216 
 217                 inst->ri_i.i_primary_ctid = *cid;
 218                 inst->ri_i.i_primary_ctid_stopped = 0;
 219 
 220                 log_framework(LOG_DEBUG, "Storing primary contract %ld for "
 221                     "%s.\n", *cid, inst->ri_i.i_fmri);
 222 
 223                 contract_hash_store(*cid, inst->ri_id);
 224         }
 225 
 226 again:
 227         if (inst->ri_mi_deleted)
 228                 return;
 229 
 230         r = restarter_store_contract(inst->ri_m_inst, *cid, primary ?
 231             RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
 232         switch (r) {
 233         case 0:
 234                 break;
 235 
 236         case ECANCELED:
 237                 inst->ri_mi_deleted = B_TRUE;
 238                 break;
 239 
 240         case ECONNABORTED:
 241                 libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
 242                 /* FALLTHROUGH */
 243 
 244         case EBADF:
 245                 libscf_reget_instance(inst);
 246                 goto again;
 247 
 248         case ENOMEM:
 249         case EPERM:
 250         case EACCES:
 251         case EROFS:
 252                 uu_die("%s: Couldn't store contract id %ld",
 253                     inst->ri_i.i_fmri, *cid);
 254                 /* NOTREACHED */
 255 
 256         case EINVAL:
 257         default:
 258                 bad_error("restarter_store_contract", r);
 259         }
 260 }
 261 
 262 /*
 263  * void method_remove_contract()
 264  *   Remove any non-permanent contracts from internal structures and
 265  *   the repository, then abandon them.
 266  *   Returns
 267  *     0 - success
 268  *     ECANCELED - inst was deleted from the repository
 269  *
 270  *   If the repository connection was broken, it is rebound.
 271  */
 272 void
 273 method_remove_contract(restarter_inst_t *inst, boolean_t primary,
 274     boolean_t abandon)
 275 {
 276         ctid_t * const ctidp = primary ? &inst->ri_i.i_primary_ctid :
 277             &inst->ri_i.i_transient_ctid;
 278 
 279         int r;
 280 
 281         assert(*ctidp != 0);
 282 
 283         log_framework(LOG_DEBUG, "Removing %s contract %lu for %s.\n",
 284             primary ? "primary" : "transient", *ctidp, inst->ri_i.i_fmri);
 285 
 286         if (abandon)
 287                 contract_abandon(*ctidp);
 288 
 289 again:
 290         if (inst->ri_mi_deleted) {
 291                 r = ECANCELED;
 292                 goto out;
 293         }
 294 
 295         r = restarter_remove_contract(inst->ri_m_inst, *ctidp, primary ?
 296             RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
 297         switch (r) {
 298         case 0:
 299                 break;
 300 
 301         case ECANCELED:
 302                 inst->ri_mi_deleted = B_TRUE;
 303                 break;
 304 
 305         case ECONNABORTED:
 306                 libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
 307                 /* FALLTHROUGH */
 308 
 309         case EBADF:
 310                 libscf_reget_instance(inst);
 311                 goto again;
 312 
 313         case ENOMEM:
 314         case EPERM:
 315         case EACCES:
 316         case EROFS:
 317                 log_error(LOG_INFO, "%s: Couldn't remove contract id %ld: "
 318                     "%s.\n", inst->ri_i.i_fmri, *ctidp, strerror(r));
 319                 break;
 320 
 321         case EINVAL:
 322         default:
 323                 bad_error("restarter_remove_contract", r);
 324         }
 325 
 326 out:
 327         if (primary)
 328                 contract_hash_remove(*ctidp);
 329 
 330         *ctidp = 0;
 331 }
 332 
 333 static const char *method_names[] = { "start", "stop", "refresh" };
 334 
 335 /*
 336  * int method_ready_contract(restarter_inst_t *, int, method_restart_t, int)
 337  *
 338  *   Activate a contract template for the type method of inst.  type,
 339  *   restart_on, and cte_mask dictate the critical events term of the contract.
 340  *   Returns
 341  *     0 - success
 342  *     ECANCELED - inst has been deleted from the repository
 343  */
 344 static int
 345 method_ready_contract(restarter_inst_t *inst, int type,
 346     method_restart_t restart_on, uint_t cte_mask)
 347 {
 348         int tmpl, err, istrans, iswait, ret;
 349         uint_t cevents, fevents;
 350 
 351         /*
 352          * Correctly supporting wait-style services is tricky without
 353          * rearchitecting startd to cope with multiple event sources
 354          * simultaneously trying to stop an instance.  Until a better
 355          * solution is implemented, we avoid this problem for
 356          * wait-style services by making contract events fatal and
 357          * letting the wait code alone handle stopping the service.
 358          */
 359         iswait = instance_is_wait_style(inst);
 360         istrans = method_is_transient(inst, type);
 361 
 362         tmpl = open64(CTFS_ROOT "/process/template", O_RDWR);
 363         if (tmpl == -1)
 364                 uu_die("Could not create contract template");
 365 
 366         /*
 367          * We assume non-login processes are unlikely to create
 368          * multiple process groups, and set CT_PR_PGRPONLY for all
 369          * wait-style services' contracts.
 370          */
 371         err = ct_pr_tmpl_set_param(tmpl, CT_PR_INHERIT | CT_PR_REGENT |
 372             (iswait ? CT_PR_PGRPONLY : 0));
 373         assert(err == 0);
 374 
 375         if (istrans) {
 376                 cevents = 0;
 377                 fevents = 0;
 378         } else {
 379                 assert(restart_on >= 0);
 380                 assert(restart_on <= METHOD_RESTART_ANY_FAULT);
 381                 cevents = method_events[restart_on] & ~cte_mask;
 382                 fevents = iswait ?
 383                     (method_events[restart_on] & ~cte_mask & CT_PR_ALLFATAL) :
 384                     0;
 385         }
 386 
 387         err = ct_tmpl_set_critical(tmpl, cevents);
 388         assert(err == 0);
 389 
 390         err = ct_tmpl_set_informative(tmpl, 0);
 391         assert(err == 0);
 392         err = ct_pr_tmpl_set_fatal(tmpl, fevents);
 393         assert(err == 0);
 394 
 395         err = ct_tmpl_set_cookie(tmpl, istrans ?  METHOD_OTHER_COOKIE :
 396             METHOD_START_COOKIE);
 397         assert(err == 0);
 398 
 399         if (type == METHOD_START && inst->ri_i.i_primary_ctid != 0) {
 400                 ret = ct_pr_tmpl_set_transfer(tmpl, inst->ri_i.i_primary_ctid);
 401                 switch (ret) {
 402                 case 0:
 403                         break;
 404 
 405                 case ENOTEMPTY:
 406                         /* No contracts for you! */
 407                         method_remove_contract(inst, B_TRUE, B_TRUE);
 408                         if (inst->ri_mi_deleted) {
 409                                 ret = ECANCELED;
 410                                 goto out;
 411                         }
 412                         break;
 413 
 414                 case EINVAL:
 415                 case ESRCH:
 416                 case EACCES:
 417                 default:
 418                         bad_error("ct_pr_tmpl_set_transfer", ret);
 419                 }
 420         }
 421 
 422         err = ct_pr_tmpl_set_svc_fmri(tmpl, inst->ri_i.i_fmri);
 423         assert(err == 0);
 424         err = ct_pr_tmpl_set_svc_aux(tmpl, method_names[type]);
 425         assert(err == 0);
 426 
 427         err = ct_tmpl_activate(tmpl);
 428         assert(err == 0);
 429 
 430         ret = 0;
 431 
 432 out:
 433         err = close(tmpl);
 434         assert(err == 0);
 435 
 436         return (ret);
 437 }
 438 
 439 static void
 440 exec_method(const restarter_inst_t *inst, int type, const char *method,
 441     struct method_context *mcp, uint8_t need_session)
 442 {
 443         char *cmd;
 444         const char *errf;
 445         char **nenv;
 446         int rsmc_errno = 0;
 447 
 448         cmd = uu_msprintf("exec %s", method);
 449 
 450         if (inst->ri_utmpx_prefix[0] != '\0' && inst->ri_utmpx_prefix != NULL)
 451                 (void) utmpx_mark_init(getpid(), inst->ri_utmpx_prefix);
 452 
 453         setlog(inst->ri_logstem);
 454         log_instance(inst, B_FALSE, "Executing %s method (\"%s\").",
 455             method_names[type], method);
 456 
 457         if (need_session)
 458                 (void) setpgrp();
 459 
 460         /* Set credentials. */
 461         rsmc_errno = restarter_set_method_context(mcp, &errf);
 462         if (rsmc_errno != 0) {
 463                 log_instance(inst, B_FALSE,
 464                     "svc.startd could not set context for method: ");
 465 
 466                 if (rsmc_errno == -1) {
 467                         if (strcmp(errf, "core_set_process_path") == 0) {
 468                                 log_instance(inst, B_FALSE,
 469                                     "Could not set corefile path.");
 470                         } else if (strcmp(errf, "setproject") == 0) {
 471                                 log_instance(inst, B_FALSE, "%s: a resource "
 472                                     "control assignment failed", errf);
 473                         } else if (strcmp(errf, "pool_set_binding") == 0) {
 474                                 log_instance(inst, B_FALSE, "%s: a system "
 475                                     "error occurred", errf);
 476                         } else {
 477 #ifndef NDEBUG
 478                                 uu_warn("%s:%d: Bad function name \"%s\" for "
 479                                     "error %d from "
 480                                     "restarter_set_method_context().\n",
 481                                     __FILE__, __LINE__, errf, rsmc_errno);
 482 #endif
 483                                 abort();
 484                         }
 485 
 486                         exit(1);
 487                 }
 488 
 489                 if (errf != NULL && strcmp(errf, "pool_set_binding") == 0) {
 490                         switch (rsmc_errno) {
 491                         case ENOENT:
 492                                 log_instance(inst, B_FALSE, "%s: the pool "
 493                                     "could not be found", errf);
 494                                 break;
 495 
 496                         case EBADF:
 497                                 log_instance(inst, B_FALSE, "%s: the "
 498                                     "configuration is invalid", errf);
 499                                 break;
 500 
 501                         case EINVAL:
 502                                 log_instance(inst, B_FALSE, "%s: pool name "
 503                                     "\"%s\" is invalid", errf,
 504                                     mcp->resource_pool);
 505                                 break;
 506 
 507                         default:
 508 #ifndef NDEBUG
 509                                 uu_warn("%s:%d: Bad error %d for function %s "
 510                                     "in restarter_set_method_context().\n",
 511                                     __FILE__, __LINE__, rsmc_errno, errf);
 512 #endif
 513                                 abort();
 514                         }
 515 
 516                         exit(SMF_EXIT_ERR_CONFIG);
 517                 }
 518 
 519                 if (errf != NULL && strcmp(errf, "chdir") == 0) {
 520                         switch (rsmc_errno) {
 521                         case EACCES:
 522                         case EFAULT:
 523                         case EIO:
 524                         case ELOOP:
 525                         case ENAMETOOLONG:
 526                         case ENOENT:
 527                         case ENOLINK:
 528                         case ENOTDIR:
 529                                 log_instance(inst, B_FALSE, "%s: %s (\"%s\")",
 530                                     errf,
 531                                     strerror(rsmc_errno), mcp->working_dir);
 532                                 break;
 533 
 534                         default:
 535 #ifndef NDEBUG
 536                                 uu_warn("%s:%d: Bad error %d for function %s "
 537                                     "in restarter_set_method_context().\n",
 538                                     __FILE__, __LINE__, rsmc_errno, errf);
 539 #endif
 540                                 abort();
 541                         }
 542 
 543                         exit(SMF_EXIT_ERR_CONFIG);
 544                 }
 545 
 546                 if (errf != NULL) {
 547                         errno = rsmc_errno;
 548                         perror(errf);
 549 
 550                         switch (rsmc_errno) {
 551                         case EINVAL:
 552                         case EPERM:
 553                         case ENOENT:
 554                         case ENAMETOOLONG:
 555                         case ERANGE:
 556                         case ESRCH:
 557                                 exit(SMF_EXIT_ERR_CONFIG);
 558                                 /* NOTREACHED */
 559 
 560                         default:
 561                                 exit(1);
 562                         }
 563                 }
 564 
 565                 switch (rsmc_errno) {
 566                 case ENOMEM:
 567                         log_instance(inst, B_FALSE, "Out of memory.");
 568                         exit(1);
 569                         /* NOTREACHED */
 570 
 571                 case ENOENT:
 572                         log_instance(inst, B_FALSE, "Missing passwd entry for "
 573                             "user.");
 574                         exit(SMF_EXIT_ERR_CONFIG);
 575                         /* NOTREACHED */
 576 
 577                 default:
 578 #ifndef NDEBUG
 579                         uu_warn("%s:%d: Bad miscellaneous error %d from "
 580                             "restarter_set_method_context().\n", __FILE__,
 581                             __LINE__, rsmc_errno);
 582 #endif
 583                         abort();
 584                 }
 585         }
 586 
 587         nenv = set_smf_env(mcp->env, mcp->env_sz, NULL, inst,
 588             method_names[type]);
 589 
 590         log_preexec();
 591 
 592         (void) execle(SBIN_SH, SBIN_SH, "-c", cmd, NULL, nenv);
 593 
 594         exit(10);
 595 }
 596 
 597 static void
 598 write_status(restarter_inst_t *inst, const char *mname, int stat)
 599 {
 600         int r;
 601 
 602 again:
 603         if (inst->ri_mi_deleted)
 604                 return;
 605 
 606         r = libscf_write_method_status(inst->ri_m_inst, mname, stat);
 607         switch (r) {
 608         case 0:
 609                 break;
 610 
 611         case ECONNABORTED:
 612                 libscf_reget_instance(inst);
 613                 goto again;
 614 
 615         case ECANCELED:
 616                 inst->ri_mi_deleted = 1;
 617                 break;
 618 
 619         case EPERM:
 620         case EACCES:
 621         case EROFS:
 622                 log_framework(LOG_INFO, "Could not write exit status "
 623                     "for %s method of %s: %s.\n", mname,
 624                     inst->ri_i.i_fmri, strerror(r));
 625                 break;
 626 
 627         case ENAMETOOLONG:
 628         default:
 629                 bad_error("libscf_write_method_status", r);
 630         }
 631 }
 632 
 633 /*
 634  * int method_run()
 635  *   Execute the type method of instp.  If it requires a fork(), wait for it
 636  *   to return and return its exit code in *exit_code.  Otherwise set
 637  *   *exit_code to 0 if the method succeeds & -1 if it fails.  If the
 638  *   repository connection is broken, it is rebound, but inst may not be
 639  *   reset.
 640  *   Returns
 641  *     0 - success
 642  *     EINVAL - A correct method or method context couldn't be retrieved.
 643  *     EIO - Contract kill failed.
 644  *     EFAULT - Method couldn't be executed successfully.
 645  *     ELOOP - Retry threshold exceeded.
 646  *     ECANCELED - inst was deleted from the repository before method was run
 647  *     ERANGE - Timeout retry threshold exceeded.
 648  *     EAGAIN - Failed due to external cause, retry.
 649  */
 650 int
 651 method_run(restarter_inst_t **instp, int type, int *exit_code)
 652 {
 653         char *method;
 654         int ret_status;
 655         pid_t pid;
 656         method_restart_t restart_on;
 657         uint_t cte_mask;
 658         uint8_t need_session;
 659         scf_handle_t *h;
 660         scf_snapshot_t *snap;
 661         const char *mname;
 662         mc_error_t *m_error;
 663         struct method_context *mcp;
 664         int result = 0, timeout_fired = 0;
 665         int sig, r;
 666         boolean_t transient;
 667         uint64_t timeout;
 668         uint8_t timeout_retry;
 669         ctid_t ctid;
 670         int ctfd = -1;
 671         restarter_inst_t *inst = *instp;
 672         int id = inst->ri_id;
 673         int forkerr;
 674 
 675         assert(MUTEX_HELD(&inst->ri_lock));
 676         assert(instance_in_transition(inst));
 677 
 678         if (inst->ri_mi_deleted)
 679                 return (ECANCELED);
 680 
 681         *exit_code = 0;
 682 
 683         assert(0 <= type && type <= 2);
 684         mname = method_names[type];
 685 
 686         if (type == METHOD_START)
 687                 inst->ri_pre_online_hook();
 688 
 689         h = scf_instance_handle(inst->ri_m_inst);
 690 
 691         snap = scf_snapshot_create(h);
 692         if (snap == NULL ||
 693             scf_instance_get_snapshot(inst->ri_m_inst, "running", snap) != 0) {
 694                 log_framework(LOG_DEBUG,
 695                     "Could not get running snapshot for %s.  "
 696                     "Using editing version to run method %s.\n",
 697                     inst->ri_i.i_fmri, mname);
 698                 scf_snapshot_destroy(snap);
 699                 snap = NULL;
 700         }
 701 
 702         /*
 703          * After this point, we may be logging to the instance log.
 704          * Make sure we've noted where that log is as a property of
 705          * the instance.
 706          */
 707         r = libscf_note_method_log(inst->ri_m_inst, st->st_log_prefix,
 708             inst->ri_logstem);
 709         if (r != 0) {
 710                 log_framework(LOG_WARNING,
 711                     "%s: couldn't note log location: %s\n",
 712                     inst->ri_i.i_fmri, strerror(r));
 713         }
 714 
 715         if ((method = libscf_get_method(h, type, inst, snap, &restart_on,
 716             &cte_mask, &need_session, &timeout, &timeout_retry)) == NULL) {
 717                 if (errno == LIBSCF_PGROUP_ABSENT)  {
 718                         log_framework(LOG_DEBUG,
 719                             "%s: instance has no method property group '%s'.\n",
 720                             inst->ri_i.i_fmri, mname);
 721                         if (type == METHOD_REFRESH)
 722                                 log_instance(inst, B_TRUE, "No '%s' method "
 723                                     "defined.  Treating as :true.", mname);
 724                         else
 725                                 log_instance(inst, B_TRUE, "Method property "
 726                                     "group '%s' is not present.", mname);
 727                         scf_snapshot_destroy(snap);
 728                         return (0);
 729                 } else if (errno == LIBSCF_PROPERTY_ABSENT)  {
 730                         log_framework(LOG_DEBUG,
 731                             "%s: instance has no '%s/exec' method property.\n",
 732                             inst->ri_i.i_fmri, mname);
 733                         log_instance(inst, B_TRUE, "Method property '%s/exec "
 734                             "is not present.", mname);
 735                         scf_snapshot_destroy(snap);
 736                         return (0);
 737                 } else {
 738                         log_error(LOG_WARNING,
 739                             "%s: instance libscf_get_method failed\n",
 740                             inst->ri_i.i_fmri);
 741                         scf_snapshot_destroy(snap);
 742                         return (EINVAL);
 743                 }
 744         }
 745 
 746         /* open service contract if stopping a non-transient service */
 747         if (type == METHOD_STOP && (!instance_is_transient_style(inst))) {
 748                 if (inst->ri_i.i_primary_ctid == 0) {
 749                         /* service is not running, nothing to stop */
 750                         log_framework(LOG_DEBUG, "%s: instance has no primary "
 751                             "contract, no service to stop.\n",
 752                             inst->ri_i.i_fmri);
 753                         scf_snapshot_destroy(snap);
 754                         return (0);
 755                 }
 756                 if ((ctfd = contract_open(inst->ri_i.i_primary_ctid, "process",
 757                     "events", O_RDONLY)) < 0) {
 758                         result = EFAULT;
 759                         log_instance(inst, B_TRUE, "Could not open service "
 760                             "contract %ld.  Stop method not run.",
 761                             inst->ri_i.i_primary_ctid);
 762                         goto out;
 763                 }
 764         }
 765 
 766         if (restarter_is_null_method(method)) {
 767                 log_framework(LOG_DEBUG, "%s: null method succeeds\n",
 768                     inst->ri_i.i_fmri);
 769 
 770                 log_instance(inst, B_TRUE, "Executing %s method (null).",
 771                     mname);
 772 
 773                 if (type == METHOD_START)
 774                         write_status(inst, mname, 0);
 775                 goto out;
 776         }
 777 
 778         sig = restarter_is_kill_method(method);
 779         if (sig >= 0) {
 780 
 781                 if (inst->ri_i.i_primary_ctid == 0) {
 782                         log_error(LOG_ERR, "%s: :kill with no contract\n",
 783                             inst->ri_i.i_fmri);
 784                         log_instance(inst, B_TRUE, "Invalid use of \":kill\" "
 785                             "as stop method for transient service.");
 786                         result = EINVAL;
 787                         goto out;
 788                 }
 789 
 790                 log_framework(LOG_DEBUG,
 791                     "%s: :killing contract with signal %d\n",
 792                     inst->ri_i.i_fmri, sig);
 793 
 794                 log_instance(inst, B_TRUE, "Executing %s method (:kill).",
 795                     mname);
 796 
 797                 if (contract_kill(inst->ri_i.i_primary_ctid, sig,
 798                     inst->ri_i.i_fmri) != 0) {
 799                         result = EIO;
 800                         goto out;
 801                 } else
 802                         goto assured_kill;
 803         }
 804 
 805         log_framework(LOG_DEBUG, "%s: forking to run method %s\n",
 806             inst->ri_i.i_fmri, method);
 807 
 808         m_error = restarter_get_method_context(RESTARTER_METHOD_CONTEXT_VERSION,
 809             inst->ri_m_inst, snap, mname, method, &mcp);
 810 
 811         if (m_error != NULL) {
 812                 log_instance(inst, B_TRUE, "%s", m_error->msg);
 813                 restarter_mc_error_destroy(m_error);
 814                 result = EINVAL;
 815                 goto out;
 816         }
 817 
 818         r = method_ready_contract(inst, type, restart_on, cte_mask);
 819         if (r != 0) {
 820                 assert(r == ECANCELED);
 821                 assert(inst->ri_mi_deleted);
 822                 restarter_free_method_context(mcp);
 823                 result = ECANCELED;
 824                 goto out;
 825         }
 826 
 827         /*
 828          * Validate safety of method contexts, to save children work.
 829          */
 830         if (!restarter_rm_libs_loadable())
 831                 log_framework(LOG_DEBUG, "%s: method contexts limited "
 832                     "to root-accessible libraries\n", inst->ri_i.i_fmri);
 833 
 834         /*
 835          * If the service is restarting too quickly, send it to
 836          * maintenance.
 837          */
 838         if (type == METHOD_START) {
 839                 method_record_start(inst);
 840                 if (method_rate_critical(inst)) {
 841                         log_instance(inst, B_TRUE, "Restarting too quickly, "
 842                             "changing state to maintenance.");
 843                         result = ELOOP;
 844                         restarter_free_method_context(mcp);
 845                         goto out;
 846                 }
 847         }
 848 
 849         atomic_add_16(&storing_contract, 1);
 850         pid = startd_fork1(&forkerr);
 851         if (pid == 0)
 852                 exec_method(inst, type, method, mcp, need_session);
 853 
 854         if (pid == -1) {
 855                 atomic_add_16(&storing_contract, -1);
 856                 if (forkerr == EAGAIN)
 857                         result = EAGAIN;
 858                 else
 859                         result = EFAULT;
 860 
 861                 log_error(LOG_WARNING,
 862                     "%s: Couldn't fork to execute method %s: %s\n",
 863                     inst->ri_i.i_fmri, method, strerror(forkerr));
 864 
 865                 restarter_free_method_context(mcp);
 866                 goto out;
 867         }
 868 
 869 
 870         /*
 871          * Get the contract id, decide whether it is primary or transient, and
 872          * stash it in inst & the repository.
 873          */
 874         method_store_contract(inst, type, &ctid);
 875         atomic_add_16(&storing_contract, -1);
 876 
 877         restarter_free_method_context(mcp);
 878 
 879         /*
 880          * Similarly for the start method PID.
 881          */
 882         if (type == METHOD_START && !inst->ri_mi_deleted)
 883                 (void) libscf_write_start_pid(inst->ri_m_inst, pid);
 884 
 885         if (instance_is_wait_style(inst) && type == METHOD_START) {
 886                 /* Wait style instances don't get timeouts on start methods. */
 887                 if (wait_register(pid, inst->ri_i.i_fmri, 1, 0)) {
 888                         log_error(LOG_WARNING,
 889                             "%s: couldn't register %ld for wait\n",
 890                             inst->ri_i.i_fmri, pid);
 891                         result = EFAULT;
 892                         goto contract_out;
 893                 }
 894                 write_status(inst, mname, 0);
 895 
 896         } else {
 897                 int r, err;
 898                 time_t start_time;
 899                 time_t end_time;
 900 
 901                 /*
 902                  * Because on upgrade/live-upgrade we may have no chance
 903                  * to override faulty timeout values on the way to
 904                  * manifest import, all services on the path to manifest
 905                  * import are treated the same as INFINITE timeout services.
 906                  */
 907 
 908                 start_time = time(NULL);
 909                 if (timeout != METHOD_TIMEOUT_INFINITE && !is_timeout_ovr(inst))
 910                         timeout_insert(inst, ctid, timeout);
 911                 else
 912                         timeout = METHOD_TIMEOUT_INFINITE;
 913 
 914                 /* Unlock the instance while waiting for the method. */
 915                 MUTEX_UNLOCK(&inst->ri_lock);
 916 
 917                 do {
 918                         r = waitpid(pid, &ret_status, NULL);
 919                 } while (r == -1 && errno == EINTR);
 920                 if (r == -1)
 921                         err = errno;
 922 
 923                 /* Re-grab the lock. */
 924                 inst = inst_lookup_by_id(id);
 925 
 926                 /*
 927                  * inst can't be removed, as the removal thread waits
 928                  * for completion of this one.
 929                  */
 930                 assert(inst != NULL);
 931                 *instp = inst;
 932 
 933                 if (inst->ri_timeout != NULL && inst->ri_timeout->te_fired)
 934                         timeout_fired = 1;
 935 
 936                 timeout_remove(inst, ctid);
 937 
 938                 log_framework(LOG_DEBUG,
 939                     "%s method for %s exited with status %d.\n", mname,
 940                     inst->ri_i.i_fmri, WEXITSTATUS(ret_status));
 941 
 942                 if (r == -1) {
 943                         log_error(LOG_WARNING,
 944                             "Couldn't waitpid() for %s method of %s (%s).\n",
 945                             mname, inst->ri_i.i_fmri, strerror(err));
 946                         result = EFAULT;
 947                         goto contract_out;
 948                 }
 949 
 950                 if (type == METHOD_START)
 951                         write_status(inst, mname, ret_status);
 952 
 953                 /* return ERANGE if this service doesn't retry on timeout */
 954                 if (timeout_fired == 1 && timeout_retry == 0) {
 955                         result = ERANGE;
 956                         goto contract_out;
 957                 }
 958 
 959                 if (!WIFEXITED(ret_status)) {
 960                         /*
 961                          * If method didn't exit itself (it was killed by an
 962                          * external entity, etc.), consider the entire
 963                          * method_run as failed.
 964                          */
 965                         if (WIFSIGNALED(ret_status)) {
 966                                 char buf[SIG2STR_MAX];
 967                                 (void) sig2str(WTERMSIG(ret_status), buf);
 968 
 969                                 log_error(LOG_WARNING, "%s: Method \"%s\" "
 970                                     "failed due to signal %s.\n",
 971                                     inst->ri_i.i_fmri, method, buf);
 972                                 log_instance(inst, B_TRUE, "Method \"%s\" "
 973                                     "failed due to signal %s.", mname, buf);
 974                         } else {
 975                                 log_error(LOG_WARNING, "%s: Method \"%s\" "
 976                                     "failed with exit status %d.\n",
 977                                     inst->ri_i.i_fmri, method,
 978                                     WEXITSTATUS(ret_status));
 979                                 log_instance(inst, B_TRUE, "Method \"%s\" "
 980                                     "failed with exit status %d.", mname,
 981                                     WEXITSTATUS(ret_status));
 982                         }
 983                         result = EAGAIN;
 984                         goto contract_out;
 985                 }
 986 
 987                 *exit_code = WEXITSTATUS(ret_status);
 988                 if (*exit_code != 0) {
 989                         log_error(LOG_WARNING,
 990                             "%s: Method \"%s\" failed with exit status %d.\n",
 991                             inst->ri_i.i_fmri, method, WEXITSTATUS(ret_status));
 992                 }
 993 
 994                 log_instance(inst, B_TRUE, "Method \"%s\" exited with status "
 995                     "%d.", mname, *exit_code);
 996 
 997                 if (*exit_code != 0)
 998                         goto contract_out;
 999 
1000                 end_time = time(NULL);
1001 
1002                 /* Give service contract remaining seconds to empty */
1003                 if (timeout != METHOD_TIMEOUT_INFINITE)
1004                         timeout -= (end_time - start_time);
1005         }
1006 
1007 assured_kill:
1008         /*
1009          * For stop methods, assure that the service contract has emptied
1010          * before returning.
1011          */
1012         if (type == METHOD_STOP && (!instance_is_transient_style(inst)) &&
1013             !(contract_is_empty(inst->ri_i.i_primary_ctid))) {
1014                 int times = 0;
1015 
1016                 if (timeout != METHOD_TIMEOUT_INFINITE)
1017                         timeout_insert(inst, inst->ri_i.i_primary_ctid,
1018                             timeout);
1019 
1020                 for (;;) {
1021                         /*
1022                          * Check frequently at first, then back off.  This
1023                          * keeps startd from idling while shutting down.
1024                          */
1025                         if (times < 20) {
1026                                 (void) poll(NULL, 0, 5);
1027                                 times++;
1028                         } else {
1029                                 (void) poll(NULL, 0, 100);
1030                         }
1031                         if (contract_is_empty(inst->ri_i.i_primary_ctid))
1032                                 break;
1033                 }
1034 
1035                 if (timeout != METHOD_TIMEOUT_INFINITE)
1036                         if (inst->ri_timeout->te_fired)
1037                                 result = EFAULT;
1038 
1039                 timeout_remove(inst, inst->ri_i.i_primary_ctid);
1040         }
1041 
1042 contract_out:
1043         /* Abandon contracts for transient methods & methods that fail. */
1044         transient = method_is_transient(inst, type);
1045         if ((transient || *exit_code != 0 || result != 0) &&
1046             (restarter_is_kill_method(method) < 0))
1047                 method_remove_contract(inst, !transient, B_TRUE);
1048 
1049 out:
1050         if (ctfd >= 0)
1051                 (void) close(ctfd);
1052         scf_snapshot_destroy(snap);
1053         free(method);
1054         return (result);
1055 }
1056 
1057 /*
1058  * The method thread executes a service method to effect a state transition.
1059  * The next_state of info->sf_id should be non-_NONE on entrance, and it will
1060  * be _NONE on exit (state will either be what next_state was (on success), or
1061  * it will be _MAINT (on error)).
1062  *
1063  * There are six classes of methods to consider: start & other (stop, refresh)
1064  * for each of "normal" services, wait services, and transient services.  For
1065  * each, the method must be fetched from the repository & executed.  fork()ed
1066  * methods must be waited on, except for the start method of wait services
1067  * (which must be registered with the wait subsystem via wait_register()).  If
1068  * the method succeeded (returned 0), then for start methods its contract
1069  * should be recorded as the primary contract for the service.  For other
1070  * methods, it should be abandoned.  If the method fails, then depending on
1071  * the failure, either the method should be reexecuted or the service should
1072  * be put into maintenance.  Either way the contract should be abandoned.
1073  */
1074 void *
1075 method_thread(void *arg)
1076 {
1077         fork_info_t *info = arg;
1078         restarter_inst_t *inst;
1079         scf_handle_t    *local_handle;
1080         scf_instance_t  *s_inst = NULL;
1081         int r, exit_code;
1082         boolean_t retryable;
1083         restarter_str_t reason;
1084 
1085         assert(0 <= info->sf_method_type && info->sf_method_type <= 2);
1086 
1087         /* Get (and lock) the restarter_inst_t. */
1088         inst = inst_lookup_by_id(info->sf_id);
1089 
1090         assert(inst->ri_method_thread != 0);
1091         assert(instance_in_transition(inst) == 1);
1092 
1093         /*
1094          * We cannot leave this function with inst in transition, because
1095          * protocol.c withholds messages for inst otherwise.
1096          */
1097 
1098         log_framework(LOG_DEBUG, "method_thread() running %s method for %s.\n",
1099             method_names[info->sf_method_type], inst->ri_i.i_fmri);
1100 
1101         local_handle = libscf_handle_create_bound_loop();
1102 
1103 rebind_retry:
1104         /* get scf_instance_t */
1105         switch (r = libscf_fmri_get_instance(local_handle, inst->ri_i.i_fmri,
1106             &s_inst)) {
1107         case 0:
1108                 break;
1109 
1110         case ECONNABORTED:
1111                 libscf_handle_rebind(local_handle);
1112                 goto rebind_retry;
1113 
1114         case ENOENT:
1115                 /*
1116                  * It's not there, but we need to call this so protocol.c
1117                  * doesn't think it's in transition anymore.
1118                  */
1119                 (void) restarter_instance_update_states(local_handle, inst,
1120                     inst->ri_i.i_state, RESTARTER_STATE_NONE, RERR_NONE,
1121                     restarter_str_none);
1122                 goto out;
1123 
1124         case EINVAL:
1125         case ENOTSUP:
1126         default:
1127                 bad_error("libscf_fmri_get_instance", r);
1128         }
1129 
1130         inst->ri_m_inst = s_inst;
1131         inst->ri_mi_deleted = B_FALSE;
1132 
1133 retry:
1134         if (info->sf_method_type == METHOD_START)
1135                 log_transition(inst, START_REQUESTED);
1136 
1137         r = method_run(&inst, info->sf_method_type, &exit_code);
1138 
1139         if (r == 0 && exit_code == 0) {
1140                 /* Success! */
1141                 assert(inst->ri_i.i_next_state != RESTARTER_STATE_NONE);
1142 
1143                 /*
1144                  * When a stop method succeeds, remove the primary contract of
1145                  * the service, unless we're going to offline, in which case
1146                  * retain the contract so we can transfer inherited contracts to
1147                  * the replacement service.
1148                  */
1149 
1150                 if (info->sf_method_type == METHOD_STOP &&
1151                     inst->ri_i.i_primary_ctid != 0) {
1152                         if (inst->ri_i.i_next_state == RESTARTER_STATE_OFFLINE)
1153                                 inst->ri_i.i_primary_ctid_stopped = 1;
1154                         else
1155                                 method_remove_contract(inst, B_TRUE, B_TRUE);
1156                 }
1157                 /*
1158                  * We don't care whether the handle was rebound because this is
1159                  * the last thing we do with it.
1160                  */
1161                 (void) restarter_instance_update_states(local_handle, inst,
1162                     inst->ri_i.i_next_state, RESTARTER_STATE_NONE,
1163                     info->sf_event_type, info->sf_reason);
1164 
1165                 (void) update_fault_count(inst, FAULT_COUNT_RESET);
1166 
1167                 goto out;
1168         }
1169 
1170         /* Failure.  Retry or go to maintenance. */
1171 
1172         if (r != 0 && r != EAGAIN) {
1173                 retryable = B_FALSE;
1174         } else {
1175                 switch (exit_code) {
1176                 case SMF_EXIT_ERR_CONFIG:
1177                 case SMF_EXIT_ERR_NOSMF:
1178                 case SMF_EXIT_ERR_PERM:
1179                 case SMF_EXIT_ERR_FATAL:
1180                         retryable = B_FALSE;
1181                         break;
1182 
1183                 default:
1184                         retryable = B_TRUE;
1185                 }
1186         }
1187 
1188         if (retryable && update_fault_count(inst, FAULT_COUNT_INCR) != 1)
1189                 goto retry;
1190 
1191         /* maintenance */
1192         if (r == ELOOP)
1193                 log_transition(inst, START_FAILED_REPEATEDLY);
1194         else if (r == ERANGE)
1195                 log_transition(inst, START_FAILED_TIMEOUT_FATAL);
1196         else if (exit_code == SMF_EXIT_ERR_CONFIG)
1197                 log_transition(inst, START_FAILED_CONFIGURATION);
1198         else if (exit_code == SMF_EXIT_ERR_FATAL)
1199                 log_transition(inst, START_FAILED_FATAL);
1200         else
1201                 log_transition(inst, START_FAILED_OTHER);
1202 
1203         if (r == ELOOP) {
1204                 reason = restarter_str_restarting_too_quickly;
1205         } else if (retryable) {
1206                 reason = restarter_str_fault_threshold_reached;
1207         } else {
1208                 reason = restarter_str_method_failed;
1209         }
1210 
1211         (void) restarter_instance_update_states(local_handle, inst,
1212             RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, RERR_FAULT,
1213             reason);
1214 
1215         if (!method_is_transient(inst, info->sf_method_type) &&
1216             inst->ri_i.i_primary_ctid != 0)
1217                 method_remove_contract(inst, B_TRUE, B_TRUE);
1218 
1219 out:
1220         inst->ri_method_thread = 0;
1221 
1222         /*
1223          * Unlock the mutex after broadcasting to avoid a race condition
1224          * with restarter_delete_inst() when the 'inst' structure is freed.
1225          */
1226         (void) pthread_cond_broadcast(&inst->ri_method_cv);
1227         MUTEX_UNLOCK(&inst->ri_lock);
1228 
1229         scf_instance_destroy(s_inst);
1230         scf_handle_destroy(local_handle);
1231         startd_free(info, sizeof (fork_info_t));
1232         return (NULL);
1233 }