1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2011 Joyent Inc.
25 */
26
27 /*
28 * method.c - method execution functions
29 *
30 * This file contains the routines needed to run a method: a fork(2)-exec(2)
31 * invocation monitored using either the contract filesystem or waitpid(2).
32 * (Plain fork1(2) support is provided in fork.c.)
33 *
34 * Contract Transfer
35 * When we restart a service, we want to transfer any contracts that the old
36 * service's contract inherited. This means that (a) we must not abandon the
37 * old contract when the service dies and (b) we must write the id of the old
38 * contract into the terms of the new contract. There should be limits to
39 * (a), though, since we don't want to keep the contract around forever. To
40 * this end we'll say that services in the offline state may have a contract
41 * to be transfered and services in the disabled or maintenance states cannot.
42 * This means that when a service transitions from online (or degraded) to
43 * offline, the contract should be preserved, and when the service transitions
44 * from offline to online (i.e., the start method), we'll transfer inherited
45 * contracts.
46 */
47
48 #include <sys/contract/process.h>
49 #include <sys/ctfs.h>
50 #include <sys/stat.h>
51 #include <sys/time.h>
52 #include <sys/types.h>
53 #include <sys/uio.h>
54 #include <sys/wait.h>
55 #include <alloca.h>
56 #include <assert.h>
57 #include <errno.h>
58 #include <fcntl.h>
59 #include <libcontract.h>
60 #include <libcontract_priv.h>
61 #include <libgen.h>
62 #include <librestart.h>
63 #include <libscf.h>
64 #include <limits.h>
65 #include <port.h>
66 #include <sac.h>
67 #include <signal.h>
68 #include <stdlib.h>
69 #include <string.h>
70 #include <strings.h>
71 #include <unistd.h>
72 #include <atomic.h>
73 #include <poll.h>
74 #include <libscf_priv.h>
75
76 #include "startd.h"
77
78 #define SBIN_SH "/sbin/sh"
79
80 /*
81 * Used to tell if contracts are in the process of being
82 * stored into the svc.startd internal hash table.
83 */
84 volatile uint16_t storing_contract = 0;
85
86 /*
87 * Mapping from restart_on method-type to contract events. Must correspond to
88 * enum method_restart_t.
89 */
90 static uint_t method_events[] = {
91 /* METHOD_RESTART_ALL */
92 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE | CT_PR_EV_EMPTY,
93 /* METHOD_RESTART_EXTERNAL_FAULT */
94 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL,
95 /* METHOD_RESTART_ANY_FAULT */
96 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE
97 };
98
99 /*
100 * method_record_start(restarter_inst_t *)
101 * Record a service start for rate limiting. Place the current time
102 * in the circular array of instance starts.
103 */
104 static void
105 method_record_start(restarter_inst_t *inst)
106 {
107 int index = inst->ri_start_index++ % RINST_START_TIMES;
108
109 inst->ri_start_time[index] = gethrtime();
110 }
111
112 /*
113 * method_rate_critical(restarter_inst_t *)
114 * Return true if the average start interval is less than the permitted
115 * interval. The implicit interval defaults to RINST_FAILURE_RATE_NS and
116 * RINST_START_TIMES but may be overridden with the svc properties
117 * startd/critical_failure_count and startd/critical_failure_period
118 * which represent the number of failures to consider and the amount of
119 * time in seconds in which that number may occur, respectively. Note that
120 * this time is measured as of the transition to 'enabled' rather than wall
121 * clock time.
122 * Implicit success if insufficient measurements for an average exist.
123 */
124 static int
125 method_rate_critical(restarter_inst_t *inst)
126 {
127 hrtime_t critical_failure_period = RINST_FAILURE_RATE_NS;
128 uint_t critical_failure_count = RINST_START_TIMES;
129 uint_t n = inst->ri_start_index;
130 hrtime_t avg_ns = 0;
131 uint64_t scf_fr, scf_st;
132 scf_propvec_t *prop = NULL;
133 scf_propvec_t restart_critical[] = {
134 { "critical_failure_period", NULL, SCF_TYPE_INTEGER, NULL, 0 },
135 { "critical_failure_count", NULL, SCF_TYPE_INTEGER, NULL, 0 },
136 { NULL }
137 };
138
139 restart_critical[0].pv_ptr = &scf_fr;
140 restart_critical[1].pv_ptr = &scf_st;
141
142 if (scf_read_propvec(inst->ri_i.i_fmri, "startd",
143 B_TRUE, restart_critical, &prop) != SCF_FAILED) {
144 /*
145 * critical_failure_period is expressed
146 * in seconds but tracked in ns
147 */
148 critical_failure_period = (hrtime_t)scf_fr * NANOSEC;
149 critical_failure_count = (uint_t)scf_st;
150 }
151 if (inst->ri_start_index < critical_failure_count)
152 return (0);
153
154 avg_ns =
155 (inst->ri_start_time[(n - 1) % critical_failure_count] -
156 inst->ri_start_time[n % critical_failure_count]) /
157 (critical_failure_count - 1);
158
159 return (avg_ns < critical_failure_period);
160 }
161
162 /*
163 * int method_is_transient()
164 * Determine if the method for the given instance is transient,
165 * from a contract perspective. Return 1 if it is, and 0 if it isn't.
166 */
167 static int
168 method_is_transient(restarter_inst_t *inst, int type)
169 {
170 if (instance_is_transient_style(inst) || type != METHOD_START)
171 return (1);
172 else
173 return (0);
174 }
175
176 /*
177 * void method_store_contract()
178 * Store the newly created contract id into local structures and
179 * the repository. If the repository connection is broken it is rebound.
180 */
181 static void
182 method_store_contract(restarter_inst_t *inst, int type, ctid_t *cid)
183 {
184 int r;
185 boolean_t primary;
186
187 if (errno = contract_latest(cid))
188 uu_die("%s: Couldn't get new contract's id", inst->ri_i.i_fmri);
189
190 primary = !method_is_transient(inst, type);
191
192 if (!primary) {
193 if (inst->ri_i.i_transient_ctid != 0) {
194 log_framework(LOG_INFO,
195 "%s: transient ctid expected to be 0 but "
196 "was set to %ld\n", inst->ri_i.i_fmri,
197 inst->ri_i.i_transient_ctid);
198 }
199
200 inst->ri_i.i_transient_ctid = *cid;
201 } else {
202 if (inst->ri_i.i_primary_ctid != 0) {
203 /*
204 * There was an old contract that we transferred.
205 * Remove it.
206 */
207 method_remove_contract(inst, B_TRUE, B_FALSE);
208 }
209
210 if (inst->ri_i.i_primary_ctid != 0) {
211 log_framework(LOG_INFO,
212 "%s: primary ctid expected to be 0 but "
213 "was set to %ld\n", inst->ri_i.i_fmri,
214 inst->ri_i.i_primary_ctid);
215 }
216
217 inst->ri_i.i_primary_ctid = *cid;
218 inst->ri_i.i_primary_ctid_stopped = 0;
219
220 log_framework(LOG_DEBUG, "Storing primary contract %ld for "
221 "%s.\n", *cid, inst->ri_i.i_fmri);
222
223 contract_hash_store(*cid, inst->ri_id);
224 }
225
226 again:
227 if (inst->ri_mi_deleted)
228 return;
229
230 r = restarter_store_contract(inst->ri_m_inst, *cid, primary ?
231 RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
232 switch (r) {
233 case 0:
234 break;
235
236 case ECANCELED:
237 inst->ri_mi_deleted = B_TRUE;
238 break;
239
240 case ECONNABORTED:
241 libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
242 /* FALLTHROUGH */
243
244 case EBADF:
245 libscf_reget_instance(inst);
246 goto again;
247
248 case ENOMEM:
249 case EPERM:
250 case EACCES:
251 case EROFS:
252 uu_die("%s: Couldn't store contract id %ld",
253 inst->ri_i.i_fmri, *cid);
254 /* NOTREACHED */
255
256 case EINVAL:
257 default:
258 bad_error("restarter_store_contract", r);
259 }
260 }
261
262 /*
263 * void method_remove_contract()
264 * Remove any non-permanent contracts from internal structures and
265 * the repository, then abandon them.
266 * Returns
267 * 0 - success
268 * ECANCELED - inst was deleted from the repository
269 *
270 * If the repository connection was broken, it is rebound.
271 */
272 void
273 method_remove_contract(restarter_inst_t *inst, boolean_t primary,
274 boolean_t abandon)
275 {
276 ctid_t * const ctidp = primary ? &inst->ri_i.i_primary_ctid :
277 &inst->ri_i.i_transient_ctid;
278
279 int r;
280
281 assert(*ctidp != 0);
282
283 log_framework(LOG_DEBUG, "Removing %s contract %lu for %s.\n",
284 primary ? "primary" : "transient", *ctidp, inst->ri_i.i_fmri);
285
286 if (abandon)
287 contract_abandon(*ctidp);
288
289 again:
290 if (inst->ri_mi_deleted) {
291 r = ECANCELED;
292 goto out;
293 }
294
295 r = restarter_remove_contract(inst->ri_m_inst, *ctidp, primary ?
296 RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
297 switch (r) {
298 case 0:
299 break;
300
301 case ECANCELED:
302 inst->ri_mi_deleted = B_TRUE;
303 break;
304
305 case ECONNABORTED:
306 libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
307 /* FALLTHROUGH */
308
309 case EBADF:
310 libscf_reget_instance(inst);
311 goto again;
312
313 case ENOMEM:
314 case EPERM:
315 case EACCES:
316 case EROFS:
317 log_error(LOG_INFO, "%s: Couldn't remove contract id %ld: "
318 "%s.\n", inst->ri_i.i_fmri, *ctidp, strerror(r));
319 break;
320
321 case EINVAL:
322 default:
323 bad_error("restarter_remove_contract", r);
324 }
325
326 out:
327 if (primary)
328 contract_hash_remove(*ctidp);
329
330 *ctidp = 0;
331 }
332
333 static const char *method_names[] = { "start", "stop", "refresh" };
334
335 /*
336 * int method_ready_contract(restarter_inst_t *, int, method_restart_t, int)
337 *
338 * Activate a contract template for the type method of inst. type,
339 * restart_on, and cte_mask dictate the critical events term of the contract.
340 * Returns
341 * 0 - success
342 * ECANCELED - inst has been deleted from the repository
343 */
344 static int
345 method_ready_contract(restarter_inst_t *inst, int type,
346 method_restart_t restart_on, uint_t cte_mask)
347 {
348 int tmpl, err, istrans, iswait, ret;
349 uint_t cevents, fevents;
350
351 /*
352 * Correctly supporting wait-style services is tricky without
353 * rearchitecting startd to cope with multiple event sources
354 * simultaneously trying to stop an instance. Until a better
355 * solution is implemented, we avoid this problem for
356 * wait-style services by making contract events fatal and
357 * letting the wait code alone handle stopping the service.
358 */
359 iswait = instance_is_wait_style(inst);
360 istrans = method_is_transient(inst, type);
361
362 tmpl = open64(CTFS_ROOT "/process/template", O_RDWR);
363 if (tmpl == -1)
364 uu_die("Could not create contract template");
365
366 /*
367 * We assume non-login processes are unlikely to create
368 * multiple process groups, and set CT_PR_PGRPONLY for all
369 * wait-style services' contracts.
370 */
371 err = ct_pr_tmpl_set_param(tmpl, CT_PR_INHERIT | CT_PR_REGENT |
372 (iswait ? CT_PR_PGRPONLY : 0));
373 assert(err == 0);
374
375 if (istrans) {
376 cevents = 0;
377 fevents = 0;
378 } else {
379 assert(restart_on >= 0);
380 assert(restart_on <= METHOD_RESTART_ANY_FAULT);
381 cevents = method_events[restart_on] & ~cte_mask;
382 fevents = iswait ?
383 (method_events[restart_on] & ~cte_mask & CT_PR_ALLFATAL) :
384 0;
385 }
386
387 err = ct_tmpl_set_critical(tmpl, cevents);
388 assert(err == 0);
389
390 err = ct_tmpl_set_informative(tmpl, 0);
391 assert(err == 0);
392 err = ct_pr_tmpl_set_fatal(tmpl, fevents);
393 assert(err == 0);
394
395 err = ct_tmpl_set_cookie(tmpl, istrans ? METHOD_OTHER_COOKIE :
396 METHOD_START_COOKIE);
397 assert(err == 0);
398
399 if (type == METHOD_START && inst->ri_i.i_primary_ctid != 0) {
400 ret = ct_pr_tmpl_set_transfer(tmpl, inst->ri_i.i_primary_ctid);
401 switch (ret) {
402 case 0:
403 break;
404
405 case ENOTEMPTY:
406 /* No contracts for you! */
407 method_remove_contract(inst, B_TRUE, B_TRUE);
408 if (inst->ri_mi_deleted) {
409 ret = ECANCELED;
410 goto out;
411 }
412 break;
413
414 case EINVAL:
415 case ESRCH:
416 case EACCES:
417 default:
418 bad_error("ct_pr_tmpl_set_transfer", ret);
419 }
420 }
421
422 err = ct_pr_tmpl_set_svc_fmri(tmpl, inst->ri_i.i_fmri);
423 assert(err == 0);
424 err = ct_pr_tmpl_set_svc_aux(tmpl, method_names[type]);
425 assert(err == 0);
426
427 err = ct_tmpl_activate(tmpl);
428 assert(err == 0);
429
430 ret = 0;
431
432 out:
433 err = close(tmpl);
434 assert(err == 0);
435
436 return (ret);
437 }
438
439 static void
440 exec_method(const restarter_inst_t *inst, int type, const char *method,
441 struct method_context *mcp, uint8_t need_session)
442 {
443 char *cmd;
444 const char *errf;
445 char **nenv;
446 int rsmc_errno = 0;
447
448 cmd = uu_msprintf("exec %s", method);
449
450 if (inst->ri_utmpx_prefix[0] != '\0' && inst->ri_utmpx_prefix != NULL)
451 (void) utmpx_mark_init(getpid(), inst->ri_utmpx_prefix);
452
453 setlog(inst->ri_logstem);
454 log_instance(inst, B_FALSE, "Executing %s method (\"%s\").",
455 method_names[type], method);
456
457 if (need_session)
458 (void) setpgrp();
459
460 /* Set credentials. */
461 rsmc_errno = restarter_set_method_context(mcp, &errf);
462 if (rsmc_errno != 0) {
463 log_instance(inst, B_FALSE,
464 "svc.startd could not set context for method: ");
465
466 if (rsmc_errno == -1) {
467 if (strcmp(errf, "core_set_process_path") == 0) {
468 log_instance(inst, B_FALSE,
469 "Could not set corefile path.");
470 } else if (strcmp(errf, "setproject") == 0) {
471 log_instance(inst, B_FALSE, "%s: a resource "
472 "control assignment failed", errf);
473 } else if (strcmp(errf, "pool_set_binding") == 0) {
474 log_instance(inst, B_FALSE, "%s: a system "
475 "error occurred", errf);
476 } else {
477 #ifndef NDEBUG
478 uu_warn("%s:%d: Bad function name \"%s\" for "
479 "error %d from "
480 "restarter_set_method_context().\n",
481 __FILE__, __LINE__, errf, rsmc_errno);
482 #endif
483 abort();
484 }
485
486 exit(1);
487 }
488
489 if (errf != NULL && strcmp(errf, "pool_set_binding") == 0) {
490 switch (rsmc_errno) {
491 case ENOENT:
492 log_instance(inst, B_FALSE, "%s: the pool "
493 "could not be found", errf);
494 break;
495
496 case EBADF:
497 log_instance(inst, B_FALSE, "%s: the "
498 "configuration is invalid", errf);
499 break;
500
501 case EINVAL:
502 log_instance(inst, B_FALSE, "%s: pool name "
503 "\"%s\" is invalid", errf,
504 mcp->resource_pool);
505 break;
506
507 default:
508 #ifndef NDEBUG
509 uu_warn("%s:%d: Bad error %d for function %s "
510 "in restarter_set_method_context().\n",
511 __FILE__, __LINE__, rsmc_errno, errf);
512 #endif
513 abort();
514 }
515
516 exit(SMF_EXIT_ERR_CONFIG);
517 }
518
519 if (errf != NULL && strcmp(errf, "chdir") == 0) {
520 switch (rsmc_errno) {
521 case EACCES:
522 case EFAULT:
523 case EIO:
524 case ELOOP:
525 case ENAMETOOLONG:
526 case ENOENT:
527 case ENOLINK:
528 case ENOTDIR:
529 log_instance(inst, B_FALSE, "%s: %s (\"%s\")",
530 errf,
531 strerror(rsmc_errno), mcp->working_dir);
532 break;
533
534 default:
535 #ifndef NDEBUG
536 uu_warn("%s:%d: Bad error %d for function %s "
537 "in restarter_set_method_context().\n",
538 __FILE__, __LINE__, rsmc_errno, errf);
539 #endif
540 abort();
541 }
542
543 exit(SMF_EXIT_ERR_CONFIG);
544 }
545
546 if (errf != NULL) {
547 errno = rsmc_errno;
548 perror(errf);
549
550 switch (rsmc_errno) {
551 case EINVAL:
552 case EPERM:
553 case ENOENT:
554 case ENAMETOOLONG:
555 case ERANGE:
556 case ESRCH:
557 exit(SMF_EXIT_ERR_CONFIG);
558 /* NOTREACHED */
559
560 default:
561 exit(1);
562 }
563 }
564
565 switch (rsmc_errno) {
566 case ENOMEM:
567 log_instance(inst, B_FALSE, "Out of memory.");
568 exit(1);
569 /* NOTREACHED */
570
571 case ENOENT:
572 log_instance(inst, B_FALSE, "Missing passwd entry for "
573 "user.");
574 exit(SMF_EXIT_ERR_CONFIG);
575 /* NOTREACHED */
576
577 default:
578 #ifndef NDEBUG
579 uu_warn("%s:%d: Bad miscellaneous error %d from "
580 "restarter_set_method_context().\n", __FILE__,
581 __LINE__, rsmc_errno);
582 #endif
583 abort();
584 }
585 }
586
587 nenv = set_smf_env(mcp->env, mcp->env_sz, NULL, inst,
588 method_names[type]);
589
590 log_preexec();
591
592 (void) execle(SBIN_SH, SBIN_SH, "-c", cmd, NULL, nenv);
593
594 exit(10);
595 }
596
597 static void
598 write_status(restarter_inst_t *inst, const char *mname, int stat)
599 {
600 int r;
601
602 again:
603 if (inst->ri_mi_deleted)
604 return;
605
606 r = libscf_write_method_status(inst->ri_m_inst, mname, stat);
607 switch (r) {
608 case 0:
609 break;
610
611 case ECONNABORTED:
612 libscf_reget_instance(inst);
613 goto again;
614
615 case ECANCELED:
616 inst->ri_mi_deleted = 1;
617 break;
618
619 case EPERM:
620 case EACCES:
621 case EROFS:
622 log_framework(LOG_INFO, "Could not write exit status "
623 "for %s method of %s: %s.\n", mname,
624 inst->ri_i.i_fmri, strerror(r));
625 break;
626
627 case ENAMETOOLONG:
628 default:
629 bad_error("libscf_write_method_status", r);
630 }
631 }
632
633 /*
634 * int method_run()
635 * Execute the type method of instp. If it requires a fork(), wait for it
636 * to return and return its exit code in *exit_code. Otherwise set
637 * *exit_code to 0 if the method succeeds & -1 if it fails. If the
638 * repository connection is broken, it is rebound, but inst may not be
639 * reset.
640 * Returns
641 * 0 - success
642 * EINVAL - A correct method or method context couldn't be retrieved.
643 * EIO - Contract kill failed.
644 * EFAULT - Method couldn't be executed successfully.
645 * ELOOP - Retry threshold exceeded.
646 * ECANCELED - inst was deleted from the repository before method was run
647 * ERANGE - Timeout retry threshold exceeded.
648 * EAGAIN - Failed due to external cause, retry.
649 */
650 int
651 method_run(restarter_inst_t **instp, int type, int *exit_code)
652 {
653 char *method;
654 int ret_status;
655 pid_t pid;
656 method_restart_t restart_on;
657 uint_t cte_mask;
658 uint8_t need_session;
659 scf_handle_t *h;
660 scf_snapshot_t *snap;
661 const char *mname;
662 mc_error_t *m_error;
663 struct method_context *mcp;
664 int result = 0, timeout_fired = 0;
665 int sig, r;
666 boolean_t transient;
667 uint64_t timeout;
668 uint8_t timeout_retry;
669 ctid_t ctid;
670 int ctfd = -1;
671 restarter_inst_t *inst = *instp;
672 int id = inst->ri_id;
673 int forkerr;
674
675 assert(MUTEX_HELD(&inst->ri_lock));
676 assert(instance_in_transition(inst));
677
678 if (inst->ri_mi_deleted)
679 return (ECANCELED);
680
681 *exit_code = 0;
682
683 assert(0 <= type && type <= 2);
684 mname = method_names[type];
685
686 if (type == METHOD_START)
687 inst->ri_pre_online_hook();
688
689 h = scf_instance_handle(inst->ri_m_inst);
690
691 snap = scf_snapshot_create(h);
692 if (snap == NULL ||
693 scf_instance_get_snapshot(inst->ri_m_inst, "running", snap) != 0) {
694 log_framework(LOG_DEBUG,
695 "Could not get running snapshot for %s. "
696 "Using editing version to run method %s.\n",
697 inst->ri_i.i_fmri, mname);
698 scf_snapshot_destroy(snap);
699 snap = NULL;
700 }
701
702 /*
703 * After this point, we may be logging to the instance log.
704 * Make sure we've noted where that log is as a property of
705 * the instance.
706 */
707 r = libscf_note_method_log(inst->ri_m_inst, st->st_log_prefix,
708 inst->ri_logstem);
709 if (r != 0) {
710 log_framework(LOG_WARNING,
711 "%s: couldn't note log location: %s\n",
712 inst->ri_i.i_fmri, strerror(r));
713 }
714
715 if ((method = libscf_get_method(h, type, inst, snap, &restart_on,
716 &cte_mask, &need_session, &timeout, &timeout_retry)) == NULL) {
717 if (errno == LIBSCF_PGROUP_ABSENT) {
718 log_framework(LOG_DEBUG,
719 "%s: instance has no method property group '%s'.\n",
720 inst->ri_i.i_fmri, mname);
721 if (type == METHOD_REFRESH)
722 log_instance(inst, B_TRUE, "No '%s' method "
723 "defined. Treating as :true.", mname);
724 else
725 log_instance(inst, B_TRUE, "Method property "
726 "group '%s' is not present.", mname);
727 scf_snapshot_destroy(snap);
728 return (0);
729 } else if (errno == LIBSCF_PROPERTY_ABSENT) {
730 log_framework(LOG_DEBUG,
731 "%s: instance has no '%s/exec' method property.\n",
732 inst->ri_i.i_fmri, mname);
733 log_instance(inst, B_TRUE, "Method property '%s/exec "
734 "is not present.", mname);
735 scf_snapshot_destroy(snap);
736 return (0);
737 } else {
738 log_error(LOG_WARNING,
739 "%s: instance libscf_get_method failed\n",
740 inst->ri_i.i_fmri);
741 scf_snapshot_destroy(snap);
742 return (EINVAL);
743 }
744 }
745
746 /* open service contract if stopping a non-transient service */
747 if (type == METHOD_STOP && (!instance_is_transient_style(inst))) {
748 if (inst->ri_i.i_primary_ctid == 0) {
749 /* service is not running, nothing to stop */
750 log_framework(LOG_DEBUG, "%s: instance has no primary "
751 "contract, no service to stop.\n",
752 inst->ri_i.i_fmri);
753 scf_snapshot_destroy(snap);
754 return (0);
755 }
756 if ((ctfd = contract_open(inst->ri_i.i_primary_ctid, "process",
757 "events", O_RDONLY)) < 0) {
758 result = EFAULT;
759 log_instance(inst, B_TRUE, "Could not open service "
760 "contract %ld. Stop method not run.",
761 inst->ri_i.i_primary_ctid);
762 goto out;
763 }
764 }
765
766 if (restarter_is_null_method(method)) {
767 log_framework(LOG_DEBUG, "%s: null method succeeds\n",
768 inst->ri_i.i_fmri);
769
770 log_instance(inst, B_TRUE, "Executing %s method (null).",
771 mname);
772
773 if (type == METHOD_START)
774 write_status(inst, mname, 0);
775 goto out;
776 }
777
778 sig = restarter_is_kill_method(method);
779 if (sig >= 0) {
780
781 if (inst->ri_i.i_primary_ctid == 0) {
782 log_error(LOG_ERR, "%s: :kill with no contract\n",
783 inst->ri_i.i_fmri);
784 log_instance(inst, B_TRUE, "Invalid use of \":kill\" "
785 "as stop method for transient service.");
786 result = EINVAL;
787 goto out;
788 }
789
790 log_framework(LOG_DEBUG,
791 "%s: :killing contract with signal %d\n",
792 inst->ri_i.i_fmri, sig);
793
794 log_instance(inst, B_TRUE, "Executing %s method (:kill).",
795 mname);
796
797 if (contract_kill(inst->ri_i.i_primary_ctid, sig,
798 inst->ri_i.i_fmri) != 0) {
799 result = EIO;
800 goto out;
801 } else
802 goto assured_kill;
803 }
804
805 log_framework(LOG_DEBUG, "%s: forking to run method %s\n",
806 inst->ri_i.i_fmri, method);
807
808 m_error = restarter_get_method_context(RESTARTER_METHOD_CONTEXT_VERSION,
809 inst->ri_m_inst, snap, mname, method, &mcp);
810
811 if (m_error != NULL) {
812 log_instance(inst, B_TRUE, "%s", m_error->msg);
813 restarter_mc_error_destroy(m_error);
814 result = EINVAL;
815 goto out;
816 }
817
818 r = method_ready_contract(inst, type, restart_on, cte_mask);
819 if (r != 0) {
820 assert(r == ECANCELED);
821 assert(inst->ri_mi_deleted);
822 restarter_free_method_context(mcp);
823 result = ECANCELED;
824 goto out;
825 }
826
827 /*
828 * Validate safety of method contexts, to save children work.
829 */
830 if (!restarter_rm_libs_loadable())
831 log_framework(LOG_DEBUG, "%s: method contexts limited "
832 "to root-accessible libraries\n", inst->ri_i.i_fmri);
833
834 /*
835 * If the service is restarting too quickly, send it to
836 * maintenance.
837 */
838 if (type == METHOD_START) {
839 method_record_start(inst);
840 if (method_rate_critical(inst)) {
841 log_instance(inst, B_TRUE, "Restarting too quickly, "
842 "changing state to maintenance.");
843 result = ELOOP;
844 restarter_free_method_context(mcp);
845 goto out;
846 }
847 }
848
849 atomic_add_16(&storing_contract, 1);
850 pid = startd_fork1(&forkerr);
851 if (pid == 0)
852 exec_method(inst, type, method, mcp, need_session);
853
854 if (pid == -1) {
855 atomic_add_16(&storing_contract, -1);
856 if (forkerr == EAGAIN)
857 result = EAGAIN;
858 else
859 result = EFAULT;
860
861 log_error(LOG_WARNING,
862 "%s: Couldn't fork to execute method %s: %s\n",
863 inst->ri_i.i_fmri, method, strerror(forkerr));
864
865 restarter_free_method_context(mcp);
866 goto out;
867 }
868
869
870 /*
871 * Get the contract id, decide whether it is primary or transient, and
872 * stash it in inst & the repository.
873 */
874 method_store_contract(inst, type, &ctid);
875 atomic_add_16(&storing_contract, -1);
876
877 restarter_free_method_context(mcp);
878
879 /*
880 * Similarly for the start method PID.
881 */
882 if (type == METHOD_START && !inst->ri_mi_deleted)
883 (void) libscf_write_start_pid(inst->ri_m_inst, pid);
884
885 if (instance_is_wait_style(inst) && type == METHOD_START) {
886 /* Wait style instances don't get timeouts on start methods. */
887 if (wait_register(pid, inst->ri_i.i_fmri, 1, 0)) {
888 log_error(LOG_WARNING,
889 "%s: couldn't register %ld for wait\n",
890 inst->ri_i.i_fmri, pid);
891 result = EFAULT;
892 goto contract_out;
893 }
894 write_status(inst, mname, 0);
895
896 } else {
897 int r, err;
898 time_t start_time;
899 time_t end_time;
900
901 /*
902 * Because on upgrade/live-upgrade we may have no chance
903 * to override faulty timeout values on the way to
904 * manifest import, all services on the path to manifest
905 * import are treated the same as INFINITE timeout services.
906 */
907
908 start_time = time(NULL);
909 if (timeout != METHOD_TIMEOUT_INFINITE && !is_timeout_ovr(inst))
910 timeout_insert(inst, ctid, timeout);
911 else
912 timeout = METHOD_TIMEOUT_INFINITE;
913
914 /* Unlock the instance while waiting for the method. */
915 MUTEX_UNLOCK(&inst->ri_lock);
916
917 do {
918 r = waitpid(pid, &ret_status, NULL);
919 } while (r == -1 && errno == EINTR);
920 if (r == -1)
921 err = errno;
922
923 /* Re-grab the lock. */
924 inst = inst_lookup_by_id(id);
925
926 /*
927 * inst can't be removed, as the removal thread waits
928 * for completion of this one.
929 */
930 assert(inst != NULL);
931 *instp = inst;
932
933 if (inst->ri_timeout != NULL && inst->ri_timeout->te_fired)
934 timeout_fired = 1;
935
936 timeout_remove(inst, ctid);
937
938 log_framework(LOG_DEBUG,
939 "%s method for %s exited with status %d.\n", mname,
940 inst->ri_i.i_fmri, WEXITSTATUS(ret_status));
941
942 if (r == -1) {
943 log_error(LOG_WARNING,
944 "Couldn't waitpid() for %s method of %s (%s).\n",
945 mname, inst->ri_i.i_fmri, strerror(err));
946 result = EFAULT;
947 goto contract_out;
948 }
949
950 if (type == METHOD_START)
951 write_status(inst, mname, ret_status);
952
953 /* return ERANGE if this service doesn't retry on timeout */
954 if (timeout_fired == 1 && timeout_retry == 0) {
955 result = ERANGE;
956 goto contract_out;
957 }
958
959 if (!WIFEXITED(ret_status)) {
960 /*
961 * If method didn't exit itself (it was killed by an
962 * external entity, etc.), consider the entire
963 * method_run as failed.
964 */
965 if (WIFSIGNALED(ret_status)) {
966 char buf[SIG2STR_MAX];
967 (void) sig2str(WTERMSIG(ret_status), buf);
968
969 log_error(LOG_WARNING, "%s: Method \"%s\" "
970 "failed due to signal %s.\n",
971 inst->ri_i.i_fmri, method, buf);
972 log_instance(inst, B_TRUE, "Method \"%s\" "
973 "failed due to signal %s.", mname, buf);
974 } else {
975 log_error(LOG_WARNING, "%s: Method \"%s\" "
976 "failed with exit status %d.\n",
977 inst->ri_i.i_fmri, method,
978 WEXITSTATUS(ret_status));
979 log_instance(inst, B_TRUE, "Method \"%s\" "
980 "failed with exit status %d.", mname,
981 WEXITSTATUS(ret_status));
982 }
983 result = EAGAIN;
984 goto contract_out;
985 }
986
987 *exit_code = WEXITSTATUS(ret_status);
988 if (*exit_code != 0) {
989 log_error(LOG_WARNING,
990 "%s: Method \"%s\" failed with exit status %d.\n",
991 inst->ri_i.i_fmri, method, WEXITSTATUS(ret_status));
992 }
993
994 log_instance(inst, B_TRUE, "Method \"%s\" exited with status "
995 "%d.", mname, *exit_code);
996
997 if (*exit_code != 0)
998 goto contract_out;
999
1000 end_time = time(NULL);
1001
1002 /* Give service contract remaining seconds to empty */
1003 if (timeout != METHOD_TIMEOUT_INFINITE)
1004 timeout -= (end_time - start_time);
1005 }
1006
1007 assured_kill:
1008 /*
1009 * For stop methods, assure that the service contract has emptied
1010 * before returning.
1011 */
1012 if (type == METHOD_STOP && (!instance_is_transient_style(inst)) &&
1013 !(contract_is_empty(inst->ri_i.i_primary_ctid))) {
1014 int times = 0;
1015
1016 if (timeout != METHOD_TIMEOUT_INFINITE)
1017 timeout_insert(inst, inst->ri_i.i_primary_ctid,
1018 timeout);
1019
1020 for (;;) {
1021 /*
1022 * Check frequently at first, then back off. This
1023 * keeps startd from idling while shutting down.
1024 */
1025 if (times < 20) {
1026 (void) poll(NULL, 0, 5);
1027 times++;
1028 } else {
1029 (void) poll(NULL, 0, 100);
1030 }
1031 if (contract_is_empty(inst->ri_i.i_primary_ctid))
1032 break;
1033 }
1034
1035 if (timeout != METHOD_TIMEOUT_INFINITE)
1036 if (inst->ri_timeout->te_fired)
1037 result = EFAULT;
1038
1039 timeout_remove(inst, inst->ri_i.i_primary_ctid);
1040 }
1041
1042 contract_out:
1043 /* Abandon contracts for transient methods & methods that fail. */
1044 transient = method_is_transient(inst, type);
1045 if ((transient || *exit_code != 0 || result != 0) &&
1046 (restarter_is_kill_method(method) < 0))
1047 method_remove_contract(inst, !transient, B_TRUE);
1048
1049 out:
1050 if (ctfd >= 0)
1051 (void) close(ctfd);
1052 scf_snapshot_destroy(snap);
1053 free(method);
1054 return (result);
1055 }
1056
1057 /*
1058 * The method thread executes a service method to effect a state transition.
1059 * The next_state of info->sf_id should be non-_NONE on entrance, and it will
1060 * be _NONE on exit (state will either be what next_state was (on success), or
1061 * it will be _MAINT (on error)).
1062 *
1063 * There are six classes of methods to consider: start & other (stop, refresh)
1064 * for each of "normal" services, wait services, and transient services. For
1065 * each, the method must be fetched from the repository & executed. fork()ed
1066 * methods must be waited on, except for the start method of wait services
1067 * (which must be registered with the wait subsystem via wait_register()). If
1068 * the method succeeded (returned 0), then for start methods its contract
1069 * should be recorded as the primary contract for the service. For other
1070 * methods, it should be abandoned. If the method fails, then depending on
1071 * the failure, either the method should be reexecuted or the service should
1072 * be put into maintenance. Either way the contract should be abandoned.
1073 */
1074 void *
1075 method_thread(void *arg)
1076 {
1077 fork_info_t *info = arg;
1078 restarter_inst_t *inst;
1079 scf_handle_t *local_handle;
1080 scf_instance_t *s_inst = NULL;
1081 int r, exit_code;
1082 boolean_t retryable;
1083 restarter_str_t reason;
1084
1085 assert(0 <= info->sf_method_type && info->sf_method_type <= 2);
1086
1087 /* Get (and lock) the restarter_inst_t. */
1088 inst = inst_lookup_by_id(info->sf_id);
1089
1090 assert(inst->ri_method_thread != 0);
1091 assert(instance_in_transition(inst) == 1);
1092
1093 /*
1094 * We cannot leave this function with inst in transition, because
1095 * protocol.c withholds messages for inst otherwise.
1096 */
1097
1098 log_framework(LOG_DEBUG, "method_thread() running %s method for %s.\n",
1099 method_names[info->sf_method_type], inst->ri_i.i_fmri);
1100
1101 local_handle = libscf_handle_create_bound_loop();
1102
1103 rebind_retry:
1104 /* get scf_instance_t */
1105 switch (r = libscf_fmri_get_instance(local_handle, inst->ri_i.i_fmri,
1106 &s_inst)) {
1107 case 0:
1108 break;
1109
1110 case ECONNABORTED:
1111 libscf_handle_rebind(local_handle);
1112 goto rebind_retry;
1113
1114 case ENOENT:
1115 /*
1116 * It's not there, but we need to call this so protocol.c
1117 * doesn't think it's in transition anymore.
1118 */
1119 (void) restarter_instance_update_states(local_handle, inst,
1120 inst->ri_i.i_state, RESTARTER_STATE_NONE, RERR_NONE,
1121 restarter_str_none);
1122 goto out;
1123
1124 case EINVAL:
1125 case ENOTSUP:
1126 default:
1127 bad_error("libscf_fmri_get_instance", r);
1128 }
1129
1130 inst->ri_m_inst = s_inst;
1131 inst->ri_mi_deleted = B_FALSE;
1132
1133 retry:
1134 if (info->sf_method_type == METHOD_START)
1135 log_transition(inst, START_REQUESTED);
1136
1137 r = method_run(&inst, info->sf_method_type, &exit_code);
1138
1139 if (r == 0 && exit_code == 0) {
1140 /* Success! */
1141 assert(inst->ri_i.i_next_state != RESTARTER_STATE_NONE);
1142
1143 /*
1144 * When a stop method succeeds, remove the primary contract of
1145 * the service, unless we're going to offline, in which case
1146 * retain the contract so we can transfer inherited contracts to
1147 * the replacement service.
1148 */
1149
1150 if (info->sf_method_type == METHOD_STOP &&
1151 inst->ri_i.i_primary_ctid != 0) {
1152 if (inst->ri_i.i_next_state == RESTARTER_STATE_OFFLINE)
1153 inst->ri_i.i_primary_ctid_stopped = 1;
1154 else
1155 method_remove_contract(inst, B_TRUE, B_TRUE);
1156 }
1157 /*
1158 * We don't care whether the handle was rebound because this is
1159 * the last thing we do with it.
1160 */
1161 (void) restarter_instance_update_states(local_handle, inst,
1162 inst->ri_i.i_next_state, RESTARTER_STATE_NONE,
1163 info->sf_event_type, info->sf_reason);
1164
1165 (void) update_fault_count(inst, FAULT_COUNT_RESET);
1166
1167 goto out;
1168 }
1169
1170 /* Failure. Retry or go to maintenance. */
1171
1172 if (r != 0 && r != EAGAIN) {
1173 retryable = B_FALSE;
1174 } else {
1175 switch (exit_code) {
1176 case SMF_EXIT_ERR_CONFIG:
1177 case SMF_EXIT_ERR_NOSMF:
1178 case SMF_EXIT_ERR_PERM:
1179 case SMF_EXIT_ERR_FATAL:
1180 retryable = B_FALSE;
1181 break;
1182
1183 default:
1184 retryable = B_TRUE;
1185 }
1186 }
1187
1188 if (retryable && update_fault_count(inst, FAULT_COUNT_INCR) != 1)
1189 goto retry;
1190
1191 /* maintenance */
1192 if (r == ELOOP)
1193 log_transition(inst, START_FAILED_REPEATEDLY);
1194 else if (r == ERANGE)
1195 log_transition(inst, START_FAILED_TIMEOUT_FATAL);
1196 else if (exit_code == SMF_EXIT_ERR_CONFIG)
1197 log_transition(inst, START_FAILED_CONFIGURATION);
1198 else if (exit_code == SMF_EXIT_ERR_FATAL)
1199 log_transition(inst, START_FAILED_FATAL);
1200 else
1201 log_transition(inst, START_FAILED_OTHER);
1202
1203 if (r == ELOOP) {
1204 reason = restarter_str_restarting_too_quickly;
1205 } else if (retryable) {
1206 reason = restarter_str_fault_threshold_reached;
1207 } else {
1208 reason = restarter_str_method_failed;
1209 }
1210
1211 (void) restarter_instance_update_states(local_handle, inst,
1212 RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, RERR_FAULT,
1213 reason);
1214
1215 if (!method_is_transient(inst, info->sf_method_type) &&
1216 inst->ri_i.i_primary_ctid != 0)
1217 method_remove_contract(inst, B_TRUE, B_TRUE);
1218
1219 out:
1220 inst->ri_method_thread = 0;
1221
1222 /*
1223 * Unlock the mutex after broadcasting to avoid a race condition
1224 * with restarter_delete_inst() when the 'inst' structure is freed.
1225 */
1226 (void) pthread_cond_broadcast(&inst->ri_method_cv);
1227 MUTEX_UNLOCK(&inst->ri_lock);
1228
1229 scf_instance_destroy(s_inst);
1230 scf_handle_destroy(local_handle);
1231 startd_free(info, sizeof (fork_info_t));
1232 return (NULL);
1233 }