1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2011 Joyent Inc.
25 */
26
27 /*
28 * method.c - method execution functions
29 *
30 * This file contains the routines needed to run a method: a fork(2)-exec(2)
31 * invocation monitored using either the contract filesystem or waitpid(2).
32 * (Plain fork1(2) support is provided in fork.c.)
33 *
34 * Contract Transfer
35 * When we restart a service, we want to transfer any contracts that the old
36 * service's contract inherited. This means that (a) we must not abandon the
37 * old contract when the service dies and (b) we must write the id of the old
38 * contract into the terms of the new contract. There should be limits to
39 * (a), though, since we don't want to keep the contract around forever. To
40 * this end we'll say that services in the offline state may have a contract
41 * to be transfered and services in the disabled or maintenance states cannot.
42 * This means that when a service transitions from online (or degraded) to
43 * offline, the contract should be preserved, and when the service transitions
44 * from offline to online (i.e., the start method), we'll transfer inherited
45 * contracts.
46 */
47
48 #include <sys/contract/process.h>
49 #include <sys/ctfs.h>
50 #include <sys/stat.h>
51 #include <sys/time.h>
52 #include <sys/types.h>
53 #include <sys/uio.h>
54 #include <sys/wait.h>
55 #include <alloca.h>
56 #include <assert.h>
57 #include <errno.h>
58 #include <fcntl.h>
59 #include <libcontract.h>
60 #include <libcontract_priv.h>
61 #include <libgen.h>
62 #include <librestart.h>
63 #include <libscf.h>
64 #include <limits.h>
65 #include <port.h>
66 #include <sac.h>
67 #include <signal.h>
68 #include <stdlib.h>
69 #include <string.h>
70 #include <strings.h>
71 #include <unistd.h>
72 #include <atomic.h>
73 #include <poll.h>
74 #include <libscf_priv.h>
75
76 #include "startd.h"
77
78 #define SBIN_SH "/sbin/sh"
79
80 /*
81 * Used to tell if contracts are in the process of being
82 * stored into the svc.startd internal hash table.
83 */
84 volatile uint16_t storing_contract = 0;
85
86 /*
87 * Mapping from restart_on method-type to contract events. Must correspond to
88 * enum method_restart_t.
89 */
90 static uint_t method_events[] = {
91 /* METHOD_RESTART_ALL */
92 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE | CT_PR_EV_EMPTY,
93 /* METHOD_RESTART_EXTERNAL_FAULT */
94 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL,
95 /* METHOD_RESTART_ANY_FAULT */
96 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE
97 };
98
99 /*
100 * method_record_start(restarter_inst_t *)
101 * Record a service start for rate limiting. Place the current time
102 * in the circular array of instance starts.
103 */
104 static void
105 method_record_start(restarter_inst_t *inst)
106 {
107 int index = inst->ri_start_index++ % RINST_START_TIMES;
108
109 inst->ri_start_time[index] = gethrtime();
110 }
111
112 /*
113 * method_rate_critical(restarter_inst_t *)
114 * Return true if the average start interval is less than the permitted
115 * interval. The implicit interval defaults to RINST_FAILURE_RATE_NS and
116 * RINST_START_TIMES but may be overridden with the svc properties
117 * startd/critical_failure_count and startd/critical_failure_period
118 * which represent the number of failures to consider and the amount of
119 * time in seconds in which that number may occur, respectively. Note that
120 * this time is measured as of the transition to 'enabled' rather than wall
121 * clock time.
122 * Implicit success if insufficient measurements for an average exist.
123 */
124 static int
125 method_rate_critical(restarter_inst_t *inst)
126 {
127 hrtime_t critical_failure_period = RINST_FAILURE_RATE_NS;
128 uint_t critical_failure_count = RINST_START_TIMES;
129 uint_t n = inst->ri_start_index;
130 hrtime_t avg_ns = 0;
131 uint64_t scf_fr, scf_st;
132 scf_propvec_t *prop = NULL;
133 scf_propvec_t restart_critical[] = {
134 { "critical_failure_period", NULL, SCF_TYPE_INTEGER, NULL, 0 },
135 { "critical_failure_count", NULL, SCF_TYPE_INTEGER, NULL, 0 },
136 { NULL }
137 };
138
139 restart_critical[0].pv_ptr = &scf_fr;
140 restart_critical[1].pv_ptr = &scf_st;
141
142 if (scf_read_propvec(inst->ri_i.i_fmri, "startd",
143 B_TRUE, restart_critical, &prop) != SCF_FAILED) {
144 /*
145 * critical_failure_period is expressed
146 * in seconds but tracked in ns
147 */
148 critical_failure_period = (hrtime_t)scf_fr * NANOSEC;
149 critical_failure_count = (uint_t)scf_st;
150 }
151 if (inst->ri_start_index < critical_failure_count)
152 return (0);
153
154 avg_ns =
155 (inst->ri_start_time[(n - 1) % critical_failure_count] -
156 inst->ri_start_time[n % critical_failure_count]) /
157 (critical_failure_count - 1);
158
159 return (avg_ns < critical_failure_period);
160 }
161
162 /*
163 * int method_is_transient()
164 * Determine if the method for the given instance is transient,
165 * from a contract perspective. Return 1 if it is, and 0 if it isn't.
166 */
167 static int
168 method_is_transient(restarter_inst_t *inst, int type)
169 {
170 if (instance_is_transient_style(inst) || type != METHOD_START)
171 return (1);
172 else
173 return (0);
174 }
175
176 /*
177 * void method_store_contract()
178 * Store the newly created contract id into local structures and
179 * the repository. If the repository connection is broken it is rebound.
180 */
181 static void
182 method_store_contract(restarter_inst_t *inst, int type, ctid_t *cid)
183 {
184 int r;
185 boolean_t primary;
186
187 if (errno = contract_latest(cid))
188 uu_die("%s: Couldn't get new contract's id", inst->ri_i.i_fmri);
189
190 primary = !method_is_transient(inst, type);
191
192 if (!primary) {
193 if (inst->ri_i.i_transient_ctid != 0) {
194 log_framework(LOG_INFO,
195 "%s: transient ctid expected to be 0 but "
196 "was set to %ld\n", inst->ri_i.i_fmri,
197 inst->ri_i.i_transient_ctid);
198 }
199
200 inst->ri_i.i_transient_ctid = *cid;
201 } else {
202 if (inst->ri_i.i_primary_ctid != 0) {
203 /*
204 * There was an old contract that we transferred.
205 * Remove it.
206 */
207 method_remove_contract(inst, B_TRUE, B_FALSE);
208 }
209
210 if (inst->ri_i.i_primary_ctid != 0) {
211 log_framework(LOG_INFO,
212 "%s: primary ctid expected to be 0 but "
213 "was set to %ld\n", inst->ri_i.i_fmri,
214 inst->ri_i.i_primary_ctid);
215 }
216
217 inst->ri_i.i_primary_ctid = *cid;
218 inst->ri_i.i_primary_ctid_stopped = 0;
219
220 log_framework(LOG_DEBUG, "Storing primary contract %ld for "
221 "%s.\n", *cid, inst->ri_i.i_fmri);
222
223 contract_hash_store(*cid, inst->ri_id);
224 }
225
226 again:
227 if (inst->ri_mi_deleted)
228 return;
229
230 r = restarter_store_contract(inst->ri_m_inst, *cid, primary ?
231 RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
232 switch (r) {
233 case 0:
234 break;
235
236 case ECANCELED:
237 inst->ri_mi_deleted = B_TRUE;
238 break;
239
240 case ECONNABORTED:
241 libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
242 /* FALLTHROUGH */
243
244 case EBADF:
245 libscf_reget_instance(inst);
246 goto again;
247
248 case ENOMEM:
249 case EPERM:
250 case EACCES:
251 case EROFS:
252 uu_die("%s: Couldn't store contract id %ld",
253 inst->ri_i.i_fmri, *cid);
254 /* NOTREACHED */
255
256 case EINVAL:
257 default:
258 bad_error("restarter_store_contract", r);
259 }
260 }
261
262 /*
263 * void method_remove_contract()
264 * Remove any non-permanent contracts from internal structures and
265 * the repository, then abandon them.
266 * Returns
267 * 0 - success
268 * ECANCELED - inst was deleted from the repository
269 *
270 * If the repository connection was broken, it is rebound.
271 */
272 void
273 method_remove_contract(restarter_inst_t *inst, boolean_t primary,
274 boolean_t abandon)
275 {
276 ctid_t * const ctidp = primary ? &inst->ri_i.i_primary_ctid :
277 &inst->ri_i.i_transient_ctid;
278
279 int r;
280
281 assert(*ctidp != 0);
282
283 log_framework(LOG_DEBUG, "Removing %s contract %lu for %s.\n",
284 primary ? "primary" : "transient", *ctidp, inst->ri_i.i_fmri);
285
286 if (abandon)
287 contract_abandon(*ctidp);
288
289 again:
290 if (inst->ri_mi_deleted) {
291 r = ECANCELED;
292 goto out;
293 }
294
295 r = restarter_remove_contract(inst->ri_m_inst, *ctidp, primary ?
296 RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
297 switch (r) {
298 case 0:
299 break;
300
301 case ECANCELED:
302 inst->ri_mi_deleted = B_TRUE;
303 break;
304
305 case ECONNABORTED:
306 libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
307 /* FALLTHROUGH */
308
309 case EBADF:
310 libscf_reget_instance(inst);
311 goto again;
312
313 case ENOMEM:
314 case EPERM:
315 case EACCES:
316 case EROFS:
317 log_error(LOG_INFO, "%s: Couldn't remove contract id %ld: "
318 "%s.\n", inst->ri_i.i_fmri, *ctidp, strerror(r));
319 break;
320
321 case EINVAL:
322 default:
323 bad_error("restarter_remove_contract", r);
324 }
325
326 out:
327 if (primary)
328 contract_hash_remove(*ctidp);
329
330 *ctidp = 0;
331 }
332
333 static const char *method_names[] = { "start", "stop", "refresh" };
334
335 /*
336 * int method_ready_contract(restarter_inst_t *, int, method_restart_t, int)
337 *
338 * Activate a contract template for the type method of inst. type,
339 * restart_on, and cte_mask dictate the critical events term of the contract.
340 * Returns
341 * 0 - success
342 * ECANCELED - inst has been deleted from the repository
343 */
344 static int
345 method_ready_contract(restarter_inst_t *inst, int type,
346 method_restart_t restart_on, uint_t cte_mask)
347 {
348 int tmpl, err, istrans, iswait, ret;
349 uint_t cevents, fevents;
350
351 /*
352 * Correctly supporting wait-style services is tricky without
353 * rearchitecting startd to cope with multiple event sources
354 * simultaneously trying to stop an instance. Until a better
355 * solution is implemented, we avoid this problem for
356 * wait-style services by making contract events fatal and
357 * letting the wait code alone handle stopping the service.
358 */
359 iswait = instance_is_wait_style(inst);
360 istrans = method_is_transient(inst, type);
361
362 tmpl = open64(CTFS_ROOT "/process/template", O_RDWR);
363 if (tmpl == -1)
364 uu_die("Could not create contract template");
365
366 /*
367 * We assume non-login processes are unlikely to create
368 * multiple process groups, and set CT_PR_PGRPONLY for all
369 * wait-style services' contracts.
370 */
371 err = ct_pr_tmpl_set_param(tmpl, CT_PR_INHERIT | CT_PR_REGENT |
372 (iswait ? CT_PR_PGRPONLY : 0));
373 assert(err == 0);
374
375 if (istrans) {
376 cevents = 0;
377 fevents = 0;
378 } else {
379 assert(restart_on >= 0);
380 assert(restart_on <= METHOD_RESTART_ANY_FAULT);
381 cevents = method_events[restart_on] & ~cte_mask;
382 fevents = iswait ?
383 (method_events[restart_on] & ~cte_mask & CT_PR_ALLFATAL) :
384 0;
385 }
386
387 err = ct_tmpl_set_critical(tmpl, cevents);
388 assert(err == 0);
389
390 err = ct_tmpl_set_informative(tmpl, 0);
391 assert(err == 0);
392 err = ct_pr_tmpl_set_fatal(tmpl, fevents);
393 assert(err == 0);
394
395 err = ct_tmpl_set_cookie(tmpl, istrans ? METHOD_OTHER_COOKIE :
396 METHOD_START_COOKIE);
397 assert(err == 0);
398
399 if (type == METHOD_START && inst->ri_i.i_primary_ctid != 0) {
400 ret = ct_pr_tmpl_set_transfer(tmpl, inst->ri_i.i_primary_ctid);
401 switch (ret) {
402 case 0:
403 break;
404
405 case ENOTEMPTY:
406 /* No contracts for you! */
407 method_remove_contract(inst, B_TRUE, B_TRUE);
408 if (inst->ri_mi_deleted) {
409 ret = ECANCELED;
410 goto out;
411 }
412 break;
413
414 case EINVAL:
415 case ESRCH:
416 case EACCES:
417 default:
418 bad_error("ct_pr_tmpl_set_transfer", ret);
419 }
420 }
421
422 err = ct_pr_tmpl_set_svc_fmri(tmpl, inst->ri_i.i_fmri);
423 assert(err == 0);
424 err = ct_pr_tmpl_set_svc_aux(tmpl, method_names[type]);
425 assert(err == 0);
426
427 err = ct_tmpl_activate(tmpl);
428 assert(err == 0);
429
430 ret = 0;
431
432 out:
433 err = close(tmpl);
434 assert(err == 0);
435
436 return (ret);
437 }
438
439 static void
440 exec_method(const restarter_inst_t *inst, int type, const char *method,
441 struct method_context *mcp, uint8_t need_session)
442 {
443 char *cmd;
444 const char *errf;
445 char **nenv;
446 int rsmc_errno = 0;
447
448 cmd = uu_msprintf("exec %s", method);
449
450 if (inst->ri_utmpx_prefix[0] != '\0' && inst->ri_utmpx_prefix != NULL)
451 (void) utmpx_mark_init(getpid(), inst->ri_utmpx_prefix);
452
453 setlog(inst->ri_logstem);
454 log_instance(inst, B_FALSE, "Executing %s method (\"%s\").",
455 method_names[type], method);
456
457 if (need_session)
458 (void) setpgrp();
459
460 /* Set credentials. */
461 rsmc_errno = restarter_set_method_context(mcp, &errf);
462 if (rsmc_errno != 0) {
463 log_instance(inst, B_FALSE,
464 "svc.startd could not set context for method: ");
465
466 if (rsmc_errno == -1) {
467 if (strcmp(errf, "core_set_process_path") == 0) {
468 log_instance(inst, B_FALSE,
469 "Could not set corefile path.");
470 } else if (strcmp(errf, "setproject") == 0) {
471 log_instance(inst, B_FALSE, "%s: a resource "
472 "control assignment failed", errf);
473 } else if (strcmp(errf, "pool_set_binding") == 0) {
474 log_instance(inst, B_FALSE, "%s: a system "
475 "error occurred", errf);
476 } else {
477 #ifndef NDEBUG
478 uu_warn("%s:%d: Bad function name \"%s\" for "
479 "error %d from "
480 "restarter_set_method_context().\n",
481 __FILE__, __LINE__, errf, rsmc_errno);
482 #endif
483 abort();
484 }
485
486 exit(1);
487 }
488
489 if (errf != NULL && strcmp(errf, "pool_set_binding") == 0) {
490 switch (rsmc_errno) {
491 case ENOENT:
492 log_instance(inst, B_FALSE, "%s: the pool "
493 "could not be found", errf);
494 break;
495
496 case EBADF:
497 log_instance(inst, B_FALSE, "%s: the "
498 "configuration is invalid", errf);
499 break;
500
501 case EINVAL:
502 log_instance(inst, B_FALSE, "%s: pool name "
503 "\"%s\" is invalid", errf,
504 mcp->resource_pool);
505 break;
506
507 default:
508 #ifndef NDEBUG
509 uu_warn("%s:%d: Bad error %d for function %s "
510 "in restarter_set_method_context().\n",
511 __FILE__, __LINE__, rsmc_errno, errf);
512 #endif
513 abort();
514 }
515
516 exit(SMF_EXIT_ERR_CONFIG);
517 }
518
519 if (errf != NULL) {
520 errno = rsmc_errno;
521 perror(errf);
522
523 switch (rsmc_errno) {
524 case EINVAL:
525 case EPERM:
526 case ENOENT:
527 case ENAMETOOLONG:
528 case ERANGE:
529 case ESRCH:
530 exit(SMF_EXIT_ERR_CONFIG);
531 /* NOTREACHED */
532
533 default:
534 exit(1);
535 }
536 }
537
538 switch (rsmc_errno) {
539 case ENOMEM:
540 log_instance(inst, B_FALSE, "Out of memory.");
541 exit(1);
542 /* NOTREACHED */
543
544 case ENOENT:
545 log_instance(inst, B_FALSE, "Missing passwd entry for "
546 "user.");
547 exit(SMF_EXIT_ERR_CONFIG);
548 /* NOTREACHED */
549
550 default:
551 #ifndef NDEBUG
552 uu_warn("%s:%d: Bad miscellaneous error %d from "
553 "restarter_set_method_context().\n", __FILE__,
554 __LINE__, rsmc_errno);
555 #endif
556 abort();
557 }
558 }
559
560 nenv = set_smf_env(mcp->env, mcp->env_sz, NULL, inst,
561 method_names[type]);
562
563 log_preexec();
564
565 (void) execle(SBIN_SH, SBIN_SH, "-c", cmd, NULL, nenv);
566
567 exit(10);
568 }
569
570 static void
571 write_status(restarter_inst_t *inst, const char *mname, int stat)
572 {
573 int r;
574
575 again:
576 if (inst->ri_mi_deleted)
577 return;
578
579 r = libscf_write_method_status(inst->ri_m_inst, mname, stat);
580 switch (r) {
581 case 0:
582 break;
583
584 case ECONNABORTED:
585 libscf_reget_instance(inst);
586 goto again;
587
588 case ECANCELED:
589 inst->ri_mi_deleted = 1;
590 break;
591
592 case EPERM:
593 case EACCES:
594 case EROFS:
595 log_framework(LOG_INFO, "Could not write exit status "
596 "for %s method of %s: %s.\n", mname,
597 inst->ri_i.i_fmri, strerror(r));
598 break;
599
600 case ENAMETOOLONG:
601 default:
602 bad_error("libscf_write_method_status", r);
603 }
604 }
605
606 /*
607 * int method_run()
608 * Execute the type method of instp. If it requires a fork(), wait for it
609 * to return and return its exit code in *exit_code. Otherwise set
610 * *exit_code to 0 if the method succeeds & -1 if it fails. If the
611 * repository connection is broken, it is rebound, but inst may not be
612 * reset.
613 * Returns
614 * 0 - success
615 * EINVAL - A correct method or method context couldn't be retrieved.
616 * EIO - Contract kill failed.
617 * EFAULT - Method couldn't be executed successfully.
618 * ELOOP - Retry threshold exceeded.
619 * ECANCELED - inst was deleted from the repository before method was run
620 * ERANGE - Timeout retry threshold exceeded.
621 * EAGAIN - Failed due to external cause, retry.
622 */
623 int
624 method_run(restarter_inst_t **instp, int type, int *exit_code)
625 {
626 char *method;
627 int ret_status;
628 pid_t pid;
629 method_restart_t restart_on;
630 uint_t cte_mask;
631 uint8_t need_session;
632 scf_handle_t *h;
633 scf_snapshot_t *snap;
634 const char *mname;
635 mc_error_t *m_error;
636 struct method_context *mcp;
637 int result = 0, timeout_fired = 0;
638 int sig, r;
639 boolean_t transient;
640 uint64_t timeout;
641 uint8_t timeout_retry;
642 ctid_t ctid;
643 int ctfd = -1;
644 restarter_inst_t *inst = *instp;
645 int id = inst->ri_id;
646 int forkerr;
647
648 assert(MUTEX_HELD(&inst->ri_lock));
649 assert(instance_in_transition(inst));
650
651 if (inst->ri_mi_deleted)
652 return (ECANCELED);
653
654 *exit_code = 0;
655
656 assert(0 <= type && type <= 2);
657 mname = method_names[type];
658
659 if (type == METHOD_START)
660 inst->ri_pre_online_hook();
661
662 h = scf_instance_handle(inst->ri_m_inst);
663
664 snap = scf_snapshot_create(h);
665 if (snap == NULL ||
666 scf_instance_get_snapshot(inst->ri_m_inst, "running", snap) != 0) {
667 log_framework(LOG_DEBUG,
668 "Could not get running snapshot for %s. "
669 "Using editing version to run method %s.\n",
670 inst->ri_i.i_fmri, mname);
671 scf_snapshot_destroy(snap);
672 snap = NULL;
673 }
674
675 /*
676 * After this point, we may be logging to the instance log.
677 * Make sure we've noted where that log is as a property of
678 * the instance.
679 */
680 r = libscf_note_method_log(inst->ri_m_inst, st->st_log_prefix,
681 inst->ri_logstem);
682 if (r != 0) {
683 log_framework(LOG_WARNING,
684 "%s: couldn't note log location: %s\n",
685 inst->ri_i.i_fmri, strerror(r));
686 }
687
688 if ((method = libscf_get_method(h, type, inst, snap, &restart_on,
689 &cte_mask, &need_session, &timeout, &timeout_retry)) == NULL) {
690 if (errno == LIBSCF_PGROUP_ABSENT) {
691 log_framework(LOG_DEBUG,
692 "%s: instance has no method property group '%s'.\n",
693 inst->ri_i.i_fmri, mname);
694 if (type == METHOD_REFRESH)
695 log_instance(inst, B_TRUE, "No '%s' method "
696 "defined. Treating as :true.", mname);
697 else
698 log_instance(inst, B_TRUE, "Method property "
699 "group '%s' is not present.", mname);
700 scf_snapshot_destroy(snap);
701 return (0);
702 } else if (errno == LIBSCF_PROPERTY_ABSENT) {
703 log_framework(LOG_DEBUG,
704 "%s: instance has no '%s/exec' method property.\n",
705 inst->ri_i.i_fmri, mname);
706 log_instance(inst, B_TRUE, "Method property '%s/exec "
707 "is not present.", mname);
708 scf_snapshot_destroy(snap);
709 return (0);
710 } else {
711 log_error(LOG_WARNING,
712 "%s: instance libscf_get_method failed\n",
713 inst->ri_i.i_fmri);
714 scf_snapshot_destroy(snap);
715 return (EINVAL);
716 }
717 }
718
719 /* open service contract if stopping a non-transient service */
720 if (type == METHOD_STOP && (!instance_is_transient_style(inst))) {
721 if (inst->ri_i.i_primary_ctid == 0) {
722 /* service is not running, nothing to stop */
723 log_framework(LOG_DEBUG, "%s: instance has no primary "
724 "contract, no service to stop.\n",
725 inst->ri_i.i_fmri);
726 scf_snapshot_destroy(snap);
727 return (0);
728 }
729 if ((ctfd = contract_open(inst->ri_i.i_primary_ctid, "process",
730 "events", O_RDONLY)) < 0) {
731 result = EFAULT;
732 log_instance(inst, B_TRUE, "Could not open service "
733 "contract %ld. Stop method not run.",
734 inst->ri_i.i_primary_ctid);
735 goto out;
736 }
737 }
738
739 if (restarter_is_null_method(method)) {
740 log_framework(LOG_DEBUG, "%s: null method succeeds\n",
741 inst->ri_i.i_fmri);
742
743 log_instance(inst, B_TRUE, "Executing %s method (null).",
744 mname);
745
746 if (type == METHOD_START)
747 write_status(inst, mname, 0);
748 goto out;
749 }
750
751 sig = restarter_is_kill_method(method);
752 if (sig >= 0) {
753
754 if (inst->ri_i.i_primary_ctid == 0) {
755 log_error(LOG_ERR, "%s: :kill with no contract\n",
756 inst->ri_i.i_fmri);
757 log_instance(inst, B_TRUE, "Invalid use of \":kill\" "
758 "as stop method for transient service.");
759 result = EINVAL;
760 goto out;
761 }
762
763 log_framework(LOG_DEBUG,
764 "%s: :killing contract with signal %d\n",
765 inst->ri_i.i_fmri, sig);
766
767 log_instance(inst, B_TRUE, "Executing %s method (:kill).",
768 mname);
769
770 if (contract_kill(inst->ri_i.i_primary_ctid, sig,
771 inst->ri_i.i_fmri) != 0) {
772 result = EIO;
773 goto out;
774 } else
775 goto assured_kill;
776 }
777
778 log_framework(LOG_DEBUG, "%s: forking to run method %s\n",
779 inst->ri_i.i_fmri, method);
780
781 m_error = restarter_get_method_context(RESTARTER_METHOD_CONTEXT_VERSION,
782 inst->ri_m_inst, snap, mname, method, &mcp);
783
784 if (m_error != NULL) {
785 log_instance(inst, B_TRUE, "%s", m_error->msg);
786 restarter_mc_error_destroy(m_error);
787 result = EINVAL;
788 goto out;
789 }
790
791 r = method_ready_contract(inst, type, restart_on, cte_mask);
792 if (r != 0) {
793 assert(r == ECANCELED);
794 assert(inst->ri_mi_deleted);
795 restarter_free_method_context(mcp);
796 result = ECANCELED;
797 goto out;
798 }
799
800 /*
801 * Validate safety of method contexts, to save children work.
802 */
803 if (!restarter_rm_libs_loadable())
804 log_framework(LOG_DEBUG, "%s: method contexts limited "
805 "to root-accessible libraries\n", inst->ri_i.i_fmri);
806
807 /*
808 * If the service is restarting too quickly, send it to
809 * maintenance.
810 */
811 if (type == METHOD_START) {
812 method_record_start(inst);
813 if (method_rate_critical(inst)) {
814 log_instance(inst, B_TRUE, "Restarting too quickly, "
815 "changing state to maintenance.");
816 result = ELOOP;
817 restarter_free_method_context(mcp);
818 goto out;
819 }
820 }
821
822 atomic_add_16(&storing_contract, 1);
823 pid = startd_fork1(&forkerr);
824 if (pid == 0)
825 exec_method(inst, type, method, mcp, need_session);
826
827 if (pid == -1) {
828 atomic_add_16(&storing_contract, -1);
829 if (forkerr == EAGAIN)
830 result = EAGAIN;
831 else
832 result = EFAULT;
833
834 log_error(LOG_WARNING,
835 "%s: Couldn't fork to execute method %s: %s\n",
836 inst->ri_i.i_fmri, method, strerror(forkerr));
837
838 restarter_free_method_context(mcp);
839 goto out;
840 }
841
842
843 /*
844 * Get the contract id, decide whether it is primary or transient, and
845 * stash it in inst & the repository.
846 */
847 method_store_contract(inst, type, &ctid);
848 atomic_add_16(&storing_contract, -1);
849
850 restarter_free_method_context(mcp);
851
852 /*
853 * Similarly for the start method PID.
854 */
855 if (type == METHOD_START && !inst->ri_mi_deleted)
856 (void) libscf_write_start_pid(inst->ri_m_inst, pid);
857
858 if (instance_is_wait_style(inst) && type == METHOD_START) {
859 /* Wait style instances don't get timeouts on start methods. */
860 if (wait_register(pid, inst->ri_i.i_fmri, 1, 0)) {
861 log_error(LOG_WARNING,
862 "%s: couldn't register %ld for wait\n",
863 inst->ri_i.i_fmri, pid);
864 result = EFAULT;
865 goto contract_out;
866 }
867 write_status(inst, mname, 0);
868
869 } else {
870 int r, err;
871 time_t start_time;
872 time_t end_time;
873
874 /*
875 * Because on upgrade/live-upgrade we may have no chance
876 * to override faulty timeout values on the way to
877 * manifest import, all services on the path to manifest
878 * import are treated the same as INFINITE timeout services.
879 */
880
881 start_time = time(NULL);
882 if (timeout != METHOD_TIMEOUT_INFINITE && !is_timeout_ovr(inst))
883 timeout_insert(inst, ctid, timeout);
884 else
885 timeout = METHOD_TIMEOUT_INFINITE;
886
887 /* Unlock the instance while waiting for the method. */
888 MUTEX_UNLOCK(&inst->ri_lock);
889
890 do {
891 r = waitpid(pid, &ret_status, NULL);
892 } while (r == -1 && errno == EINTR);
893 if (r == -1)
894 err = errno;
895
896 /* Re-grab the lock. */
897 inst = inst_lookup_by_id(id);
898
899 /*
900 * inst can't be removed, as the removal thread waits
901 * for completion of this one.
902 */
903 assert(inst != NULL);
904 *instp = inst;
905
906 if (inst->ri_timeout != NULL && inst->ri_timeout->te_fired)
907 timeout_fired = 1;
908
909 timeout_remove(inst, ctid);
910
911 log_framework(LOG_DEBUG,
912 "%s method for %s exited with status %d.\n", mname,
913 inst->ri_i.i_fmri, WEXITSTATUS(ret_status));
914
915 if (r == -1) {
916 log_error(LOG_WARNING,
917 "Couldn't waitpid() for %s method of %s (%s).\n",
918 mname, inst->ri_i.i_fmri, strerror(err));
919 result = EFAULT;
920 goto contract_out;
921 }
922
923 if (type == METHOD_START)
924 write_status(inst, mname, ret_status);
925
926 /* return ERANGE if this service doesn't retry on timeout */
927 if (timeout_fired == 1 && timeout_retry == 0) {
928 result = ERANGE;
929 goto contract_out;
930 }
931
932 if (!WIFEXITED(ret_status)) {
933 /*
934 * If method didn't exit itself (it was killed by an
935 * external entity, etc.), consider the entire
936 * method_run as failed.
937 */
938 if (WIFSIGNALED(ret_status)) {
939 char buf[SIG2STR_MAX];
940 (void) sig2str(WTERMSIG(ret_status), buf);
941
942 log_error(LOG_WARNING, "%s: Method \"%s\" "
943 "failed due to signal %s.\n",
944 inst->ri_i.i_fmri, method, buf);
945 log_instance(inst, B_TRUE, "Method \"%s\" "
946 "failed due to signal %s.", mname, buf);
947 } else {
948 log_error(LOG_WARNING, "%s: Method \"%s\" "
949 "failed with exit status %d.\n",
950 inst->ri_i.i_fmri, method,
951 WEXITSTATUS(ret_status));
952 log_instance(inst, B_TRUE, "Method \"%s\" "
953 "failed with exit status %d.", mname,
954 WEXITSTATUS(ret_status));
955 }
956 result = EAGAIN;
957 goto contract_out;
958 }
959
960 *exit_code = WEXITSTATUS(ret_status);
961 if (*exit_code != 0) {
962 log_error(LOG_WARNING,
963 "%s: Method \"%s\" failed with exit status %d.\n",
964 inst->ri_i.i_fmri, method, WEXITSTATUS(ret_status));
965 }
966
967 log_instance(inst, B_TRUE, "Method \"%s\" exited with status "
968 "%d.", mname, *exit_code);
969
970 if (*exit_code != 0)
971 goto contract_out;
972
973 end_time = time(NULL);
974
975 /* Give service contract remaining seconds to empty */
976 if (timeout != METHOD_TIMEOUT_INFINITE)
977 timeout -= (end_time - start_time);
978 }
979
980 assured_kill:
981 /*
982 * For stop methods, assure that the service contract has emptied
983 * before returning.
984 */
985 if (type == METHOD_STOP && (!instance_is_transient_style(inst)) &&
986 !(contract_is_empty(inst->ri_i.i_primary_ctid))) {
987 int times = 0;
988
989 if (timeout != METHOD_TIMEOUT_INFINITE)
990 timeout_insert(inst, inst->ri_i.i_primary_ctid,
991 timeout);
992
993 for (;;) {
994 /*
995 * Check frequently at first, then back off. This
996 * keeps startd from idling while shutting down.
997 */
998 if (times < 20) {
999 (void) poll(NULL, 0, 5);
1000 times++;
1001 } else {
1002 (void) poll(NULL, 0, 100);
1003 }
1004 if (contract_is_empty(inst->ri_i.i_primary_ctid))
1005 break;
1006 }
1007
1008 if (timeout != METHOD_TIMEOUT_INFINITE)
1009 if (inst->ri_timeout->te_fired)
1010 result = EFAULT;
1011
1012 timeout_remove(inst, inst->ri_i.i_primary_ctid);
1013 }
1014
1015 contract_out:
1016 /* Abandon contracts for transient methods & methods that fail. */
1017 transient = method_is_transient(inst, type);
1018 if ((transient || *exit_code != 0 || result != 0) &&
1019 (restarter_is_kill_method(method) < 0))
1020 method_remove_contract(inst, !transient, B_TRUE);
1021
1022 out:
1023 if (ctfd >= 0)
1024 (void) close(ctfd);
1025 scf_snapshot_destroy(snap);
1026 free(method);
1027 return (result);
1028 }
1029
1030 /*
1031 * The method thread executes a service method to effect a state transition.
1032 * The next_state of info->sf_id should be non-_NONE on entrance, and it will
1033 * be _NONE on exit (state will either be what next_state was (on success), or
1034 * it will be _MAINT (on error)).
1035 *
1036 * There are six classes of methods to consider: start & other (stop, refresh)
1037 * for each of "normal" services, wait services, and transient services. For
1038 * each, the method must be fetched from the repository & executed. fork()ed
1039 * methods must be waited on, except for the start method of wait services
1040 * (which must be registered with the wait subsystem via wait_register()). If
1041 * the method succeeded (returned 0), then for start methods its contract
1042 * should be recorded as the primary contract for the service. For other
1043 * methods, it should be abandoned. If the method fails, then depending on
1044 * the failure, either the method should be reexecuted or the service should
1045 * be put into maintenance. Either way the contract should be abandoned.
1046 */
1047 void *
1048 method_thread(void *arg)
1049 {
1050 fork_info_t *info = arg;
1051 restarter_inst_t *inst;
1052 scf_handle_t *local_handle;
1053 scf_instance_t *s_inst = NULL;
1054 int r, exit_code;
1055 boolean_t retryable;
1056 restarter_str_t reason;
1057
1058 assert(0 <= info->sf_method_type && info->sf_method_type <= 2);
1059
1060 /* Get (and lock) the restarter_inst_t. */
1061 inst = inst_lookup_by_id(info->sf_id);
1062
1063 assert(inst->ri_method_thread != 0);
1064 assert(instance_in_transition(inst) == 1);
1065
1066 /*
1067 * We cannot leave this function with inst in transition, because
1068 * protocol.c withholds messages for inst otherwise.
1069 */
1070
1071 log_framework(LOG_DEBUG, "method_thread() running %s method for %s.\n",
1072 method_names[info->sf_method_type], inst->ri_i.i_fmri);
1073
1074 local_handle = libscf_handle_create_bound_loop();
1075
1076 rebind_retry:
1077 /* get scf_instance_t */
1078 switch (r = libscf_fmri_get_instance(local_handle, inst->ri_i.i_fmri,
1079 &s_inst)) {
1080 case 0:
1081 break;
1082
1083 case ECONNABORTED:
1084 libscf_handle_rebind(local_handle);
1085 goto rebind_retry;
1086
1087 case ENOENT:
1088 /*
1089 * It's not there, but we need to call this so protocol.c
1090 * doesn't think it's in transition anymore.
1091 */
1092 (void) restarter_instance_update_states(local_handle, inst,
1093 inst->ri_i.i_state, RESTARTER_STATE_NONE, RERR_NONE,
1094 restarter_str_none);
1095 goto out;
1096
1097 case EINVAL:
1098 case ENOTSUP:
1099 default:
1100 bad_error("libscf_fmri_get_instance", r);
1101 }
1102
1103 inst->ri_m_inst = s_inst;
1104 inst->ri_mi_deleted = B_FALSE;
1105
1106 retry:
1107 if (info->sf_method_type == METHOD_START)
1108 log_transition(inst, START_REQUESTED);
1109
1110 r = method_run(&inst, info->sf_method_type, &exit_code);
1111
1112 if (r == 0 && exit_code == 0) {
1113 /* Success! */
1114 assert(inst->ri_i.i_next_state != RESTARTER_STATE_NONE);
1115
1116 /*
1117 * When a stop method succeeds, remove the primary contract of
1118 * the service, unless we're going to offline, in which case
1119 * retain the contract so we can transfer inherited contracts to
1120 * the replacement service.
1121 */
1122
1123 if (info->sf_method_type == METHOD_STOP &&
1124 inst->ri_i.i_primary_ctid != 0) {
1125 if (inst->ri_i.i_next_state == RESTARTER_STATE_OFFLINE)
1126 inst->ri_i.i_primary_ctid_stopped = 1;
1127 else
1128 method_remove_contract(inst, B_TRUE, B_TRUE);
1129 }
1130 /*
1131 * We don't care whether the handle was rebound because this is
1132 * the last thing we do with it.
1133 */
1134 (void) restarter_instance_update_states(local_handle, inst,
1135 inst->ri_i.i_next_state, RESTARTER_STATE_NONE,
1136 info->sf_event_type, info->sf_reason);
1137
1138 (void) update_fault_count(inst, FAULT_COUNT_RESET);
1139
1140 goto out;
1141 }
1142
1143 /* Failure. Retry or go to maintenance. */
1144
1145 if (r != 0 && r != EAGAIN) {
1146 retryable = B_FALSE;
1147 } else {
1148 switch (exit_code) {
1149 case SMF_EXIT_ERR_CONFIG:
1150 case SMF_EXIT_ERR_NOSMF:
1151 case SMF_EXIT_ERR_PERM:
1152 case SMF_EXIT_ERR_FATAL:
1153 retryable = B_FALSE;
1154 break;
1155
1156 default:
1157 retryable = B_TRUE;
1158 }
1159 }
1160
1161 if (retryable && update_fault_count(inst, FAULT_COUNT_INCR) != 1)
1162 goto retry;
1163
1164 /* maintenance */
1165 if (r == ELOOP)
1166 log_transition(inst, START_FAILED_REPEATEDLY);
1167 else if (r == ERANGE)
1168 log_transition(inst, START_FAILED_TIMEOUT_FATAL);
1169 else if (exit_code == SMF_EXIT_ERR_CONFIG)
1170 log_transition(inst, START_FAILED_CONFIGURATION);
1171 else if (exit_code == SMF_EXIT_ERR_FATAL)
1172 log_transition(inst, START_FAILED_FATAL);
1173 else
1174 log_transition(inst, START_FAILED_OTHER);
1175
1176 if (r == ELOOP) {
1177 reason = restarter_str_restarting_too_quickly;
1178 } else if (retryable) {
1179 reason = restarter_str_fault_threshold_reached;
1180 } else {
1181 reason = restarter_str_method_failed;
1182 }
1183
1184 (void) restarter_instance_update_states(local_handle, inst,
1185 RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, RERR_FAULT,
1186 reason);
1187
1188 if (!method_is_transient(inst, info->sf_method_type) &&
1189 inst->ri_i.i_primary_ctid != 0)
1190 method_remove_contract(inst, B_TRUE, B_TRUE);
1191
1192 out:
1193 inst->ri_method_thread = 0;
1194
1195 /*
1196 * Unlock the mutex after broadcasting to avoid a race condition
1197 * with restarter_delete_inst() when the 'inst' structure is freed.
1198 */
1199 (void) pthread_cond_broadcast(&inst->ri_method_cv);
1200 MUTEX_UNLOCK(&inst->ri_lock);
1201
1202 scf_instance_destroy(s_inst);
1203 scf_handle_destroy(local_handle);
1204 startd_free(info, sizeof (fork_info_t));
1205 return (NULL);
1206 }