1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2011 Joyent Inc.
25 * Copyright 2017 RackTop Systems.
26 */
27
28 /*
29 * method.c - method execution functions
30 *
31 * This file contains the routines needed to run a method: a fork(2)-exec(2)
32 * invocation monitored using either the contract filesystem or waitpid(2).
33 * (Plain fork1(2) support is provided in fork.c.)
34 *
35 * Contract Transfer
36 * When we restart a service, we want to transfer any contracts that the old
37 * service's contract inherited. This means that (a) we must not abandon the
38 * old contract when the service dies and (b) we must write the id of the old
39 * contract into the terms of the new contract. There should be limits to
40 * (a), though, since we don't want to keep the contract around forever. To
41 * this end we'll say that services in the offline state may have a contract
42 * to be transfered and services in the disabled or maintenance states cannot.
43 * This means that when a service transitions from online (or degraded) to
44 * offline, the contract should be preserved, and when the service transitions
45 * from offline to online (i.e., the start method), we'll transfer inherited
46 * contracts.
47 */
48
49 #include <sys/contract/process.h>
50 #include <sys/ctfs.h>
51 #include <sys/stat.h>
52 #include <sys/time.h>
53 #include <sys/types.h>
54 #include <sys/uio.h>
55 #include <sys/wait.h>
56 #include <alloca.h>
57 #include <assert.h>
58 #include <errno.h>
59 #include <fcntl.h>
60 #include <libcontract.h>
61 #include <libcontract_priv.h>
62 #include <libgen.h>
63 #include <librestart.h>
64 #include <libscf.h>
65 #include <limits.h>
66 #include <port.h>
67 #include <sac.h>
68 #include <signal.h>
69 #include <stdlib.h>
70 #include <string.h>
71 #include <strings.h>
72 #include <unistd.h>
73 #include <atomic.h>
74 #include <poll.h>
75 #include <libscf_priv.h>
76
77 #include "startd.h"
78
79 #define SBIN_SH "/sbin/sh"
80
81 /*
82 * Used to tell if contracts are in the process of being
83 * stored into the svc.startd internal hash table.
84 */
85 volatile uint16_t storing_contract = 0;
86
87 /*
88 * Mapping from restart_on method-type to contract events. Must correspond to
89 * enum method_restart_t.
90 */
91 static uint_t method_events[] = {
92 /* METHOD_RESTART_ALL */
93 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE | CT_PR_EV_EMPTY,
94 /* METHOD_RESTART_EXTERNAL_FAULT */
95 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL,
96 /* METHOD_RESTART_ANY_FAULT */
97 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE
98 };
99
100 /*
101 * method_record_start(restarter_inst_t *)
102 * Record a service start for rate limiting. Place the current time
103 * in the circular array of instance starts.
104 */
105 static void
106 method_record_start(restarter_inst_t *inst)
107 {
108 int index = inst->ri_start_index++ % RINST_START_TIMES;
109
110 inst->ri_start_time[index] = gethrtime();
111 }
112
113 /*
114 * method_rate_critical(restarter_inst_t *)
115 * Return true if the average start interval is less than the permitted
116 * interval. The implicit interval defaults to RINST_FAILURE_RATE_NS and
117 * RINST_START_TIMES but may be overridden with the svc properties
118 * startd/critical_failure_count and startd/critical_failure_period
119 * which represent the number of failures to consider and the amount of
120 * time in seconds in which that number may occur, respectively. Note that
121 * this time is measured as of the transition to 'enabled' rather than wall
122 * clock time.
123 * Implicit success if insufficient measurements for an average exist.
124 */
125 int
126 method_rate_critical(restarter_inst_t *inst)
127 {
128 hrtime_t critical_failure_period;
129 uint_t critical_failure_count = RINST_START_TIMES;
130 uint_t n = inst->ri_start_index;
131 hrtime_t avg_ns = 0;
132 uint64_t scf_fr, scf_st;
133 scf_propvec_t *prop = NULL;
134 scf_propvec_t restart_critical[] = {
135 { "critical_failure_period", NULL, SCF_TYPE_INTEGER, NULL, 0 },
136 { "critical_failure_count", NULL, SCF_TYPE_INTEGER, NULL, 0 },
137 { NULL }
138 };
139
140 if (instance_is_wait_style(inst))
141 critical_failure_period = RINST_WT_SVC_FAILURE_RATE_NS;
142 else
143 critical_failure_period = RINST_FAILURE_RATE_NS;
144
145 restart_critical[0].pv_ptr = &scf_fr;
146 restart_critical[1].pv_ptr = &scf_st;
147
148 if (scf_read_propvec(inst->ri_i.i_fmri, "startd",
149 B_TRUE, restart_critical, &prop) != SCF_FAILED) {
150 /*
151 * critical_failure_period is expressed
152 * in seconds but tracked in ns
153 */
154 critical_failure_period = (hrtime_t)scf_fr * NANOSEC;
155 critical_failure_count = (uint_t)scf_st;
156 }
157 if (inst->ri_start_index < critical_failure_count)
158 return (0);
159
160 avg_ns =
161 (inst->ri_start_time[(n - 1) % critical_failure_count] -
162 inst->ri_start_time[n % critical_failure_count]) /
163 (critical_failure_count - 1);
164
165 return (avg_ns < critical_failure_period);
166 }
167
168 /*
169 * int method_is_transient()
170 * Determine if the method for the given instance is transient,
171 * from a contract perspective. Return 1 if it is, and 0 if it isn't.
172 */
173 static int
174 method_is_transient(restarter_inst_t *inst, int type)
175 {
176 if (instance_is_transient_style(inst) || type != METHOD_START)
177 return (1);
178 else
179 return (0);
180 }
181
182 /*
183 * int method_failed()
184 * Return 1 if the exit_code indicates failure (not all non-zero
185 * exit codes do) otherwise return 0.
186 */
187 static int
188 method_failed(int exit_code)
189 {
190 if (exit_code != 0 && exit_code != SMF_EXIT_TEMP_TRANSIENT)
191 return (1);
192 else
193 return (0);
194 }
195
196 /*
197 * void method_store_contract()
198 * Store the newly created contract id into local structures and
199 * the repository. If the repository connection is broken it is rebound.
200 */
201 static void
202 method_store_contract(restarter_inst_t *inst, int type, ctid_t *cid)
203 {
204 int r;
205 boolean_t primary;
206
207 if (errno = contract_latest(cid))
208 uu_die("%s: Couldn't get new contract's id", inst->ri_i.i_fmri);
209
210 primary = !method_is_transient(inst, type);
211
212 if (!primary) {
213 if (inst->ri_i.i_transient_ctid != 0) {
214 log_framework(LOG_INFO,
215 "%s: transient ctid expected to be 0 but "
216 "was set to %ld\n", inst->ri_i.i_fmri,
217 inst->ri_i.i_transient_ctid);
218 }
219
220 inst->ri_i.i_transient_ctid = *cid;
221 } else {
222 if (inst->ri_i.i_primary_ctid != 0) {
223 /*
224 * There was an old contract that we transferred.
225 * Remove it.
226 */
227 method_remove_contract(inst, B_TRUE, B_FALSE);
228 }
229
230 if (inst->ri_i.i_primary_ctid != 0) {
231 log_framework(LOG_INFO,
232 "%s: primary ctid expected to be 0 but "
233 "was set to %ld\n", inst->ri_i.i_fmri,
234 inst->ri_i.i_primary_ctid);
235 }
236
237 inst->ri_i.i_primary_ctid = *cid;
238 inst->ri_i.i_primary_ctid_stopped = 0;
239
240 log_framework(LOG_DEBUG, "Storing primary contract %ld for "
241 "%s.\n", *cid, inst->ri_i.i_fmri);
242
243 contract_hash_store(*cid, inst->ri_id);
244 }
245
246 again:
247 if (inst->ri_mi_deleted)
248 return;
249
250 r = restarter_store_contract(inst->ri_m_inst, *cid, primary ?
251 RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
252 switch (r) {
253 case 0:
254 break;
255
256 case ECANCELED:
257 inst->ri_mi_deleted = B_TRUE;
258 break;
259
260 case ECONNABORTED:
261 libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
262 /* FALLTHROUGH */
263
264 case EBADF:
265 libscf_reget_instance(inst);
266 goto again;
267
268 case ENOMEM:
269 case EPERM:
270 case EACCES:
271 case EROFS:
272 uu_die("%s: Couldn't store contract id %ld",
273 inst->ri_i.i_fmri, *cid);
274 /* NOTREACHED */
275
276 case EINVAL:
277 default:
278 bad_error("restarter_store_contract", r);
279 }
280 }
281
282 /*
283 * void method_remove_contract()
284 * Remove any non-permanent contracts from internal structures and
285 * the repository, then abandon them.
286 * Returns
287 * 0 - success
288 * ECANCELED - inst was deleted from the repository
289 *
290 * If the repository connection was broken, it is rebound.
291 */
292 void
293 method_remove_contract(restarter_inst_t *inst, boolean_t primary,
294 boolean_t abandon)
295 {
296 ctid_t * const ctidp = primary ? &inst->ri_i.i_primary_ctid :
297 &inst->ri_i.i_transient_ctid;
298
299 int r;
300
301 assert(*ctidp != 0);
302
303 log_framework(LOG_DEBUG, "Removing %s contract %lu for %s.\n",
304 primary ? "primary" : "transient", *ctidp, inst->ri_i.i_fmri);
305
306 if (abandon)
307 contract_abandon(*ctidp);
308
309 again:
310 if (inst->ri_mi_deleted) {
311 r = ECANCELED;
312 goto out;
313 }
314
315 r = restarter_remove_contract(inst->ri_m_inst, *ctidp, primary ?
316 RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
317 switch (r) {
318 case 0:
319 break;
320
321 case ECANCELED:
322 inst->ri_mi_deleted = B_TRUE;
323 break;
324
325 case ECONNABORTED:
326 libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
327 /* FALLTHROUGH */
328
329 case EBADF:
330 libscf_reget_instance(inst);
331 goto again;
332
333 case ENOMEM:
334 case EPERM:
335 case EACCES:
336 case EROFS:
337 log_error(LOG_INFO, "%s: Couldn't remove contract id %ld: "
338 "%s.\n", inst->ri_i.i_fmri, *ctidp, strerror(r));
339 break;
340
341 case EINVAL:
342 default:
343 bad_error("restarter_remove_contract", r);
344 }
345
346 out:
347 if (primary)
348 contract_hash_remove(*ctidp);
349
350 *ctidp = 0;
351 }
352
353 static const char *method_names[] = { "start", "stop", "refresh" };
354
355 /*
356 * int method_ready_contract(restarter_inst_t *, int, method_restart_t, int)
357 *
358 * Activate a contract template for the type method of inst. type,
359 * restart_on, and cte_mask dictate the critical events term of the contract.
360 * Returns
361 * 0 - success
362 * ECANCELED - inst has been deleted from the repository
363 */
364 static int
365 method_ready_contract(restarter_inst_t *inst, int type,
366 method_restart_t restart_on, uint_t cte_mask)
367 {
368 int tmpl, err, istrans, iswait, ret;
369 uint_t cevents, fevents;
370
371 /*
372 * Correctly supporting wait-style services is tricky without
373 * rearchitecting startd to cope with multiple event sources
374 * simultaneously trying to stop an instance. Until a better
375 * solution is implemented, we avoid this problem for
376 * wait-style services by making contract events fatal and
377 * letting the wait code alone handle stopping the service.
378 */
379 iswait = instance_is_wait_style(inst);
380 istrans = method_is_transient(inst, type);
381
382 tmpl = open64(CTFS_ROOT "/process/template", O_RDWR);
383 if (tmpl == -1)
384 uu_die("Could not create contract template");
385
386 /*
387 * We assume non-login processes are unlikely to create
388 * multiple process groups, and set CT_PR_PGRPONLY for all
389 * wait-style services' contracts.
390 */
391 err = ct_pr_tmpl_set_param(tmpl, CT_PR_INHERIT | CT_PR_REGENT |
392 (iswait ? CT_PR_PGRPONLY : 0));
393 assert(err == 0);
394
395 if (istrans) {
396 cevents = 0;
397 fevents = 0;
398 } else {
399 assert(restart_on >= 0);
400 assert(restart_on <= METHOD_RESTART_ANY_FAULT);
401 cevents = method_events[restart_on] & ~cte_mask;
402 fevents = iswait ?
403 (method_events[restart_on] & ~cte_mask & CT_PR_ALLFATAL) :
404 0;
405 }
406
407 err = ct_tmpl_set_critical(tmpl, cevents);
408 assert(err == 0);
409
410 err = ct_tmpl_set_informative(tmpl, 0);
411 assert(err == 0);
412 err = ct_pr_tmpl_set_fatal(tmpl, fevents);
413 assert(err == 0);
414
415 err = ct_tmpl_set_cookie(tmpl, istrans ? METHOD_OTHER_COOKIE :
416 METHOD_START_COOKIE);
417 assert(err == 0);
418
419 if (type == METHOD_START && inst->ri_i.i_primary_ctid != 0) {
420 ret = ct_pr_tmpl_set_transfer(tmpl, inst->ri_i.i_primary_ctid);
421 switch (ret) {
422 case 0:
423 break;
424
425 case ENOTEMPTY:
426 /* No contracts for you! */
427 method_remove_contract(inst, B_TRUE, B_TRUE);
428 if (inst->ri_mi_deleted) {
429 ret = ECANCELED;
430 goto out;
431 }
432 break;
433
434 case EINVAL:
435 case ESRCH:
436 case EACCES:
437 default:
438 bad_error("ct_pr_tmpl_set_transfer", ret);
439 }
440 }
441
442 err = ct_pr_tmpl_set_svc_fmri(tmpl, inst->ri_i.i_fmri);
443 assert(err == 0);
444 err = ct_pr_tmpl_set_svc_aux(tmpl, method_names[type]);
445 assert(err == 0);
446
447 err = ct_tmpl_activate(tmpl);
448 assert(err == 0);
449
450 ret = 0;
451
452 out:
453 err = close(tmpl);
454 assert(err == 0);
455
456 return (ret);
457 }
458
459 static void
460 exec_method(const restarter_inst_t *inst, int type, const char *method,
461 struct method_context *mcp, uint8_t need_session)
462 {
463 char *cmd;
464 const char *errf;
465 char **nenv;
466 int rsmc_errno = 0;
467
468 cmd = uu_msprintf("exec %s", method);
469
470 if (inst->ri_utmpx_prefix[0] != '\0' && inst->ri_utmpx_prefix != NULL)
471 (void) utmpx_mark_init(getpid(), inst->ri_utmpx_prefix);
472
473 setlog(inst->ri_logstem);
474 log_instance(inst, B_FALSE, "Executing %s method (\"%s\").",
475 method_names[type], method);
476
477 if (need_session)
478 (void) setpgrp();
479
480 /* Set credentials. */
481 rsmc_errno = restarter_set_method_context(mcp, &errf);
482 if (rsmc_errno != 0) {
483 log_instance(inst, B_FALSE,
484 "svc.startd could not set context for method: ");
485
486 if (rsmc_errno == -1) {
487 if (strcmp(errf, "core_set_process_path") == 0) {
488 log_instance(inst, B_FALSE,
489 "Could not set corefile path.");
490 } else if (strcmp(errf, "setproject") == 0) {
491 log_instance(inst, B_FALSE, "%s: a resource "
492 "control assignment failed", errf);
493 } else if (strcmp(errf, "pool_set_binding") == 0) {
494 log_instance(inst, B_FALSE, "%s: a system "
495 "error occurred", errf);
496 } else {
497 #ifndef NDEBUG
498 uu_warn("%s:%d: Bad function name \"%s\" for "
499 "error %d from "
500 "restarter_set_method_context().\n",
501 __FILE__, __LINE__, errf, rsmc_errno);
502 #endif
503 abort();
504 }
505
506 exit(1);
507 }
508
509 if (errf != NULL && strcmp(errf, "pool_set_binding") == 0) {
510 switch (rsmc_errno) {
511 case ENOENT:
512 log_instance(inst, B_FALSE, "%s: the pool "
513 "could not be found", errf);
514 break;
515
516 case EBADF:
517 log_instance(inst, B_FALSE, "%s: the "
518 "configuration is invalid", errf);
519 break;
520
521 case EINVAL:
522 log_instance(inst, B_FALSE, "%s: pool name "
523 "\"%s\" is invalid", errf,
524 mcp->resource_pool);
525 break;
526
527 default:
528 #ifndef NDEBUG
529 uu_warn("%s:%d: Bad error %d for function %s "
530 "in restarter_set_method_context().\n",
531 __FILE__, __LINE__, rsmc_errno, errf);
532 #endif
533 abort();
534 }
535
536 exit(SMF_EXIT_ERR_CONFIG);
537 }
538
539 if (errf != NULL && strcmp(errf, "chdir") == 0) {
540 switch (rsmc_errno) {
541 case EACCES:
542 case EFAULT:
543 case EIO:
544 case ELOOP:
545 case ENAMETOOLONG:
546 case ENOENT:
547 case ENOLINK:
548 case ENOTDIR:
549 log_instance(inst, B_FALSE, "%s: %s (\"%s\")",
550 errf,
551 strerror(rsmc_errno), mcp->working_dir);
552 break;
553
554 default:
555 #ifndef NDEBUG
556 uu_warn("%s:%d: Bad error %d for function %s "
557 "in restarter_set_method_context().\n",
558 __FILE__, __LINE__, rsmc_errno, errf);
559 #endif
560 abort();
561 }
562
563 exit(SMF_EXIT_ERR_CONFIG);
564 }
565
566 if (errf != NULL) {
567 errno = rsmc_errno;
568 perror(errf);
569
570 switch (rsmc_errno) {
571 case EINVAL:
572 case EPERM:
573 case ENOENT:
574 case ENAMETOOLONG:
575 case ERANGE:
576 case ESRCH:
577 exit(SMF_EXIT_ERR_CONFIG);
578 /* NOTREACHED */
579
580 default:
581 exit(1);
582 }
583 }
584
585 switch (rsmc_errno) {
586 case ENOMEM:
587 log_instance(inst, B_FALSE, "Out of memory.");
588 exit(1);
589 /* NOTREACHED */
590
591 case ENOENT:
592 log_instance(inst, B_FALSE, "Missing passwd entry for "
593 "user.");
594 exit(SMF_EXIT_ERR_CONFIG);
595 /* NOTREACHED */
596
597 default:
598 #ifndef NDEBUG
599 uu_warn("%s:%d: Bad miscellaneous error %d from "
600 "restarter_set_method_context().\n", __FILE__,
601 __LINE__, rsmc_errno);
602 #endif
603 abort();
604 }
605 }
606
607 nenv = set_smf_env(mcp->env, mcp->env_sz, NULL, inst,
608 method_names[type]);
609
610 log_preexec();
611
612 (void) execle(SBIN_SH, SBIN_SH, "-c", cmd, NULL, nenv);
613
614 exit(10);
615 }
616
617 static void
618 write_status(restarter_inst_t *inst, const char *mname, int stat)
619 {
620 int r;
621
622 again:
623 if (inst->ri_mi_deleted)
624 return;
625
626 r = libscf_write_method_status(inst->ri_m_inst, mname, stat);
627 switch (r) {
628 case 0:
629 break;
630
631 case ECONNABORTED:
632 libscf_reget_instance(inst);
633 goto again;
634
635 case ECANCELED:
636 inst->ri_mi_deleted = 1;
637 break;
638
639 case EPERM:
640 case EACCES:
641 case EROFS:
642 log_framework(LOG_INFO, "Could not write exit status "
643 "for %s method of %s: %s.\n", mname,
644 inst->ri_i.i_fmri, strerror(r));
645 break;
646
647 case ENAMETOOLONG:
648 default:
649 bad_error("libscf_write_method_status", r);
650 }
651 }
652
653 /*
654 * int method_run()
655 * Execute the type method of instp. If it requires a fork(), wait for it
656 * to return and return its exit code in *exit_code. Otherwise set
657 * *exit_code to 0 if the method succeeds & -1 if it fails. If the
658 * repository connection is broken, it is rebound, but inst may not be
659 * reset.
660 * Returns
661 * 0 - success
662 * EINVAL - A correct method or method context couldn't be retrieved.
663 * EIO - Contract kill failed.
664 * EFAULT - Method couldn't be executed successfully.
665 * ELOOP - Retry threshold exceeded.
666 * ECANCELED - inst was deleted from the repository before method was run
667 * ERANGE - Timeout retry threshold exceeded.
668 * EAGAIN - Failed due to external cause, retry.
669 */
670 int
671 method_run(restarter_inst_t **instp, int type, int *exit_code)
672 {
673 char *method;
674 int ret_status;
675 pid_t pid;
676 method_restart_t restart_on;
677 uint_t cte_mask;
678 uint8_t need_session;
679 scf_handle_t *h;
680 scf_snapshot_t *snap;
681 const char *mname;
682 mc_error_t *m_error;
683 struct method_context *mcp;
684 int result = 0, timeout_fired = 0;
685 int sig, r;
686 boolean_t transient;
687 uint64_t timeout;
688 uint8_t timeout_retry;
689 ctid_t ctid;
690 int ctfd = -1;
691 restarter_inst_t *inst = *instp;
692 int id = inst->ri_id;
693 int forkerr;
694
695 assert(MUTEX_HELD(&inst->ri_lock));
696 assert(instance_in_transition(inst));
697
698 if (inst->ri_mi_deleted)
699 return (ECANCELED);
700
701 *exit_code = 0;
702
703 assert(0 <= type && type <= 2);
704 mname = method_names[type];
705
706 if (type == METHOD_START)
707 inst->ri_pre_online_hook();
708
709 h = scf_instance_handle(inst->ri_m_inst);
710
711 snap = scf_snapshot_create(h);
712 if (snap == NULL ||
713 scf_instance_get_snapshot(inst->ri_m_inst, "running", snap) != 0) {
714 log_framework(LOG_DEBUG,
715 "Could not get running snapshot for %s. "
716 "Using editing version to run method %s.\n",
717 inst->ri_i.i_fmri, mname);
718 scf_snapshot_destroy(snap);
719 snap = NULL;
720 }
721
722 /*
723 * After this point, we may be logging to the instance log.
724 * Make sure we've noted where that log is as a property of
725 * the instance.
726 */
727 r = libscf_note_method_log(inst->ri_m_inst, st->st_log_prefix,
728 inst->ri_logstem);
729 if (r != 0) {
730 log_framework(LOG_WARNING,
731 "%s: couldn't note log location: %s\n",
732 inst->ri_i.i_fmri, strerror(r));
733 }
734
735 if ((method = libscf_get_method(h, type, inst, snap, &restart_on,
736 &cte_mask, &need_session, &timeout, &timeout_retry)) == NULL) {
737 if (errno == LIBSCF_PGROUP_ABSENT) {
738 log_framework(LOG_DEBUG,
739 "%s: instance has no method property group '%s'.\n",
740 inst->ri_i.i_fmri, mname);
741 if (type == METHOD_REFRESH)
742 log_instance(inst, B_TRUE, "No '%s' method "
743 "defined. Treating as :true.", mname);
744 else
745 log_instance(inst, B_TRUE, "Method property "
746 "group '%s' is not present.", mname);
747 scf_snapshot_destroy(snap);
748 return (0);
749 } else if (errno == LIBSCF_PROPERTY_ABSENT) {
750 log_framework(LOG_DEBUG,
751 "%s: instance has no '%s/exec' method property.\n",
752 inst->ri_i.i_fmri, mname);
753 log_instance(inst, B_TRUE, "Method property '%s/exec "
754 "is not present.", mname);
755 scf_snapshot_destroy(snap);
756 return (0);
757 } else {
758 log_error(LOG_WARNING,
759 "%s: instance libscf_get_method failed\n",
760 inst->ri_i.i_fmri);
761 scf_snapshot_destroy(snap);
762 return (EINVAL);
763 }
764 }
765
766 /* open service contract if stopping a non-transient service */
767 if (type == METHOD_STOP && (!instance_is_transient_style(inst))) {
768 if (inst->ri_i.i_primary_ctid == 0) {
769 /* service is not running, nothing to stop */
770 log_framework(LOG_DEBUG, "%s: instance has no primary "
771 "contract, no service to stop.\n",
772 inst->ri_i.i_fmri);
773 scf_snapshot_destroy(snap);
774 return (0);
775 }
776 if ((ctfd = contract_open(inst->ri_i.i_primary_ctid, "process",
777 "events", O_RDONLY)) < 0) {
778 result = EFAULT;
779 log_instance(inst, B_TRUE, "Could not open service "
780 "contract %ld. Stop method not run.",
781 inst->ri_i.i_primary_ctid);
782 goto out;
783 }
784 }
785
786 if (restarter_is_null_method(method)) {
787 log_framework(LOG_DEBUG, "%s: null method succeeds\n",
788 inst->ri_i.i_fmri);
789
790 log_instance(inst, B_TRUE, "Executing %s method (null).",
791 mname);
792
793 if (type == METHOD_START)
794 write_status(inst, mname, 0);
795 goto out;
796 }
797
798 sig = restarter_is_kill_method(method);
799 if (sig >= 0) {
800
801 if (inst->ri_i.i_primary_ctid == 0) {
802 log_error(LOG_ERR, "%s: :kill with no contract\n",
803 inst->ri_i.i_fmri);
804 log_instance(inst, B_TRUE, "Invalid use of \":kill\" "
805 "as stop method for transient service.");
806 result = EINVAL;
807 goto out;
808 }
809
810 log_framework(LOG_DEBUG,
811 "%s: :killing contract with signal %d\n",
812 inst->ri_i.i_fmri, sig);
813
814 log_instance(inst, B_TRUE, "Executing %s method (:kill).",
815 mname);
816
817 if (contract_kill(inst->ri_i.i_primary_ctid, sig,
818 inst->ri_i.i_fmri) != 0) {
819 result = EIO;
820 goto out;
821 } else
822 goto assured_kill;
823 }
824
825 log_framework(LOG_DEBUG, "%s: forking to run method %s\n",
826 inst->ri_i.i_fmri, method);
827
828 m_error = restarter_get_method_context(RESTARTER_METHOD_CONTEXT_VERSION,
829 inst->ri_m_inst, snap, mname, method, &mcp);
830
831 if (m_error != NULL) {
832 log_instance(inst, B_TRUE, "%s", m_error->msg);
833 restarter_mc_error_destroy(m_error);
834 result = EINVAL;
835 goto out;
836 }
837
838 r = method_ready_contract(inst, type, restart_on, cte_mask);
839 if (r != 0) {
840 assert(r == ECANCELED);
841 assert(inst->ri_mi_deleted);
842 restarter_free_method_context(mcp);
843 result = ECANCELED;
844 goto out;
845 }
846
847 /*
848 * Validate safety of method contexts, to save children work.
849 */
850 if (!restarter_rm_libs_loadable())
851 log_framework(LOG_DEBUG, "%s: method contexts limited "
852 "to root-accessible libraries\n", inst->ri_i.i_fmri);
853
854 /*
855 * For wait-style svc, sanity check that method exists to prevent an
856 * infinite loop.
857 */
858 if (instance_is_wait_style(inst) && type == METHOD_START) {
859 char *pend;
860 struct stat64 sbuf;
861
862 /*
863 * We need to handle start method strings that have arguments,
864 * such as '/lib/svc/method/console-login %i'.
865 */
866 if ((pend = strchr(method, ' ')) != NULL)
867 *pend = '\0';
868
869 if (*method == '/' && stat64(method, &sbuf) == -1 &&
870 errno == ENOENT) {
871 log_instance(inst, B_TRUE, "Missing start method (%s), "
872 "changing state to maintenance.", method);
873 restarter_free_method_context(mcp);
874 result = ENOENT;
875 goto out;
876 }
877 if (pend != NULL)
878 *pend = ' ';
879 }
880
881 /*
882 * If the service is restarting too quickly, send it to
883 * maintenance.
884 */
885 if (type == METHOD_START) {
886 method_record_start(inst);
887 if (method_rate_critical(inst) &&
888 !instance_is_wait_style(inst)) {
889 log_instance(inst, B_TRUE, "Restarting too quickly, "
890 "changing state to maintenance.");
891 result = ELOOP;
892 restarter_free_method_context(mcp);
893 goto out;
894 }
895 }
896
897 atomic_add_16(&storing_contract, 1);
898 pid = startd_fork1(&forkerr);
899 if (pid == 0)
900 exec_method(inst, type, method, mcp, need_session);
901
902 if (pid == -1) {
903 atomic_add_16(&storing_contract, -1);
904 if (forkerr == EAGAIN)
905 result = EAGAIN;
906 else
907 result = EFAULT;
908
909 log_error(LOG_WARNING,
910 "%s: Couldn't fork to execute method %s: %s\n",
911 inst->ri_i.i_fmri, method, strerror(forkerr));
912
913 restarter_free_method_context(mcp);
914 goto out;
915 }
916
917
918 /*
919 * Get the contract id, decide whether it is primary or transient, and
920 * stash it in inst & the repository.
921 */
922 method_store_contract(inst, type, &ctid);
923 atomic_add_16(&storing_contract, -1);
924
925 restarter_free_method_context(mcp);
926
927 /*
928 * Similarly for the start method PID.
929 */
930 if (type == METHOD_START && !inst->ri_mi_deleted)
931 (void) libscf_write_start_pid(inst->ri_m_inst, pid);
932
933 if (instance_is_wait_style(inst) && type == METHOD_START) {
934 /* Wait style instances don't get timeouts on start methods. */
935 if (wait_register(pid, inst->ri_i.i_fmri, 1, 0)) {
936 log_error(LOG_WARNING,
937 "%s: couldn't register %ld for wait\n",
938 inst->ri_i.i_fmri, pid);
939 result = EFAULT;
940 goto contract_out;
941 }
942 write_status(inst, mname, 0);
943
944 } else {
945 int r, err;
946 time_t start_time;
947 time_t end_time;
948
949 /*
950 * Because on upgrade/live-upgrade we may have no chance
951 * to override faulty timeout values on the way to
952 * manifest import, all services on the path to manifest
953 * import are treated the same as INFINITE timeout services.
954 */
955
956 start_time = time(NULL);
957 if (timeout != METHOD_TIMEOUT_INFINITE && !is_timeout_ovr(inst))
958 timeout_insert(inst, ctid, timeout);
959 else
960 timeout = METHOD_TIMEOUT_INFINITE;
961
962 /* Unlock the instance while waiting for the method. */
963 MUTEX_UNLOCK(&inst->ri_lock);
964
965 do {
966 r = waitpid(pid, &ret_status, NULL);
967 } while (r == -1 && errno == EINTR);
968 if (r == -1)
969 err = errno;
970
971 /* Re-grab the lock. */
972 inst = inst_lookup_by_id(id);
973
974 /*
975 * inst can't be removed, as the removal thread waits
976 * for completion of this one.
977 */
978 assert(inst != NULL);
979 *instp = inst;
980
981 if (inst->ri_timeout != NULL && inst->ri_timeout->te_fired)
982 timeout_fired = 1;
983
984 timeout_remove(inst, ctid);
985
986 log_framework(LOG_DEBUG,
987 "%s method for %s exited with status %d.\n", mname,
988 inst->ri_i.i_fmri, WEXITSTATUS(ret_status));
989
990 if (r == -1) {
991 log_error(LOG_WARNING,
992 "Couldn't waitpid() for %s method of %s (%s).\n",
993 mname, inst->ri_i.i_fmri, strerror(err));
994 result = EFAULT;
995 goto contract_out;
996 }
997
998 if (type == METHOD_START)
999 write_status(inst, mname, ret_status);
1000
1001 /* return ERANGE if this service doesn't retry on timeout */
1002 if (timeout_fired == 1 && timeout_retry == 0) {
1003 result = ERANGE;
1004 goto contract_out;
1005 }
1006
1007 if (!WIFEXITED(ret_status)) {
1008 /*
1009 * If method didn't exit itself (it was killed by an
1010 * external entity, etc.), consider the entire
1011 * method_run as failed.
1012 */
1013 if (WIFSIGNALED(ret_status)) {
1014 char buf[SIG2STR_MAX];
1015 (void) sig2str(WTERMSIG(ret_status), buf);
1016
1017 log_error(LOG_WARNING, "%s: Method \"%s\" "
1018 "failed due to signal %s.\n",
1019 inst->ri_i.i_fmri, method, buf);
1020 log_instance(inst, B_TRUE, "Method \"%s\" "
1021 "failed due to signal %s.", mname, buf);
1022 } else {
1023 log_error(LOG_WARNING, "%s: Method \"%s\" "
1024 "failed with exit status %d.\n",
1025 inst->ri_i.i_fmri, method,
1026 WEXITSTATUS(ret_status));
1027 log_instance(inst, B_TRUE, "Method \"%s\" "
1028 "failed with exit status %d.", mname,
1029 WEXITSTATUS(ret_status));
1030 }
1031 result = EAGAIN;
1032 goto contract_out;
1033 }
1034
1035 *exit_code = WEXITSTATUS(ret_status);
1036 if (method_failed(*exit_code) != 0) {
1037 log_error(LOG_WARNING,
1038 "%s: Method \"%s\" failed with exit status %d.\n",
1039 inst->ri_i.i_fmri, method, WEXITSTATUS(ret_status));
1040 }
1041
1042 if (type == METHOD_STOP &&
1043 *exit_code == SMF_EXIT_TEMP_TRANSIENT) {
1044 log_instance(inst, B_TRUE, "Invalid use of "
1045 "\"$SMF_EXIT_TEMP_TRANSIENT\" in stop method.");
1046 result = EINVAL;
1047 goto contract_out;
1048 }
1049
1050 log_instance(inst, B_TRUE, "Method \"%s\" exited with status "
1051 "%d.", mname, *exit_code);
1052
1053 if (method_failed(*exit_code) != 0)
1054 goto contract_out;
1055
1056 end_time = time(NULL);
1057
1058 /* Give service contract remaining seconds to empty */
1059 if (timeout != METHOD_TIMEOUT_INFINITE)
1060 timeout -= (end_time - start_time);
1061 }
1062
1063 assured_kill:
1064 /*
1065 * For stop methods, assure that the service contract has emptied
1066 * before returning.
1067 */
1068 if (type == METHOD_STOP && (!instance_is_transient_style(inst)) &&
1069 !(contract_is_empty(inst->ri_i.i_primary_ctid))) {
1070 int times = 0;
1071
1072 if (timeout != METHOD_TIMEOUT_INFINITE)
1073 timeout_insert(inst, inst->ri_i.i_primary_ctid,
1074 timeout);
1075
1076 for (;;) {
1077 /*
1078 * Check frequently at first, then back off. This
1079 * keeps startd from idling while shutting down.
1080 */
1081 if (times < 20) {
1082 (void) poll(NULL, 0, 5);
1083 times++;
1084 } else {
1085 (void) poll(NULL, 0, 100);
1086 }
1087 if (contract_is_empty(inst->ri_i.i_primary_ctid))
1088 break;
1089 }
1090
1091 if (timeout != METHOD_TIMEOUT_INFINITE)
1092 if (inst->ri_timeout->te_fired)
1093 result = EFAULT;
1094
1095 timeout_remove(inst, inst->ri_i.i_primary_ctid);
1096 }
1097
1098 contract_out:
1099 /* Abandon contracts for transient methods & methods that fail. */
1100 transient = method_is_transient(inst, type);
1101 if ((transient || *exit_code != 0 || result != 0) &&
1102 (restarter_is_kill_method(method) < 0))
1103 method_remove_contract(inst, !transient, B_TRUE);
1104
1105 out:
1106 if (ctfd >= 0)
1107 (void) close(ctfd);
1108 scf_snapshot_destroy(snap);
1109 free(method);
1110 return (result);
1111 }
1112
1113 /*
1114 * The method thread executes a service method to effect a state transition.
1115 * The next_state of info->sf_id should be non-_NONE on entrance, and it will
1116 * be _NONE on exit (state will either be what next_state was (on success), or
1117 * it will be _MAINT (on error)).
1118 *
1119 * There are six classes of methods to consider: start & other (stop, refresh)
1120 * for each of "normal" services, wait services, and transient services. For
1121 * each, the method must be fetched from the repository & executed. fork()ed
1122 * methods must be waited on, except for the start method of wait services
1123 * (which must be registered with the wait subsystem via wait_register()). If
1124 * the method succeeded (returned 0), then for start methods its contract
1125 * should be recorded as the primary contract for the service. For other
1126 * methods, it should be abandoned. If the method fails, then depending on
1127 * the failure, either the method should be reexecuted or the service should
1128 * be put into maintenance. Either way the contract should be abandoned.
1129 */
1130 void *
1131 method_thread(void *arg)
1132 {
1133 fork_info_t *info = arg;
1134 restarter_inst_t *inst;
1135 scf_handle_t *local_handle;
1136 scf_instance_t *s_inst = NULL;
1137 int r, exit_code;
1138 boolean_t retryable;
1139 restarter_str_t reason;
1140
1141 assert(0 <= info->sf_method_type && info->sf_method_type <= 2);
1142
1143 /* Get (and lock) the restarter_inst_t. */
1144 inst = inst_lookup_by_id(info->sf_id);
1145
1146 assert(inst->ri_method_thread != 0);
1147 assert(instance_in_transition(inst) == 1);
1148
1149 /*
1150 * We cannot leave this function with inst in transition, because
1151 * protocol.c withholds messages for inst otherwise.
1152 */
1153
1154 log_framework(LOG_DEBUG, "method_thread() running %s method for %s.\n",
1155 method_names[info->sf_method_type], inst->ri_i.i_fmri);
1156
1157 local_handle = libscf_handle_create_bound_loop();
1158
1159 rebind_retry:
1160 /* get scf_instance_t */
1161 switch (r = libscf_fmri_get_instance(local_handle, inst->ri_i.i_fmri,
1162 &s_inst)) {
1163 case 0:
1164 break;
1165
1166 case ECONNABORTED:
1167 libscf_handle_rebind(local_handle);
1168 goto rebind_retry;
1169
1170 case ENOENT:
1171 /*
1172 * It's not there, but we need to call this so protocol.c
1173 * doesn't think it's in transition anymore.
1174 */
1175 (void) restarter_instance_update_states(local_handle, inst,
1176 inst->ri_i.i_state, RESTARTER_STATE_NONE, RERR_NONE,
1177 restarter_str_none);
1178 goto out;
1179
1180 case EINVAL:
1181 case ENOTSUP:
1182 default:
1183 bad_error("libscf_fmri_get_instance", r);
1184 }
1185
1186 inst->ri_m_inst = s_inst;
1187 inst->ri_mi_deleted = B_FALSE;
1188
1189 retry:
1190 if (info->sf_method_type == METHOD_START)
1191 log_transition(inst, START_REQUESTED);
1192
1193 r = method_run(&inst, info->sf_method_type, &exit_code);
1194
1195 if (r == 0 && method_failed(exit_code) == 0) {
1196 /* Success! */
1197 assert(inst->ri_i.i_next_state != RESTARTER_STATE_NONE);
1198
1199 /*
1200 * When a stop method succeeds, remove the primary contract of
1201 * the service, unless we're going to offline, in which case
1202 * retain the contract so we can transfer inherited contracts to
1203 * the replacement service.
1204 */
1205
1206 if (info->sf_method_type == METHOD_STOP &&
1207 inst->ri_i.i_primary_ctid != 0) {
1208 if (inst->ri_i.i_next_state == RESTARTER_STATE_OFFLINE)
1209 inst->ri_i.i_primary_ctid_stopped = 1;
1210 else
1211 method_remove_contract(inst, B_TRUE, B_TRUE);
1212 }
1213 /*
1214 * We don't care whether the handle was rebound because this is
1215 * the last thing we do with it.
1216 */
1217 (void) restarter_instance_update_states(local_handle, inst,
1218 inst->ri_i.i_next_state, RESTARTER_STATE_NONE,
1219 info->sf_event_type, info->sf_reason);
1220
1221 (void) update_fault_count(inst, FAULT_COUNT_RESET);
1222
1223 goto out;
1224 }
1225
1226 /* Failure. Retry or go to maintenance. */
1227
1228 if (r != 0 && r != EAGAIN) {
1229 retryable = B_FALSE;
1230 } else {
1231 switch (exit_code) {
1232 case SMF_EXIT_ERR_CONFIG:
1233 case SMF_EXIT_ERR_NOSMF:
1234 case SMF_EXIT_ERR_PERM:
1235 case SMF_EXIT_ERR_FATAL:
1236 retryable = B_FALSE;
1237 break;
1238
1239 default:
1240 retryable = B_TRUE;
1241 }
1242 }
1243
1244 if (retryable && update_fault_count(inst, FAULT_COUNT_INCR) != 1)
1245 goto retry;
1246
1247 /* maintenance */
1248 if (r == ELOOP)
1249 log_transition(inst, START_FAILED_REPEATEDLY);
1250 else if (r == ERANGE)
1251 log_transition(inst, START_FAILED_TIMEOUT_FATAL);
1252 else if (exit_code == SMF_EXIT_ERR_CONFIG)
1253 log_transition(inst, START_FAILED_CONFIGURATION);
1254 else if (exit_code == SMF_EXIT_ERR_FATAL)
1255 log_transition(inst, START_FAILED_FATAL);
1256 else
1257 log_transition(inst, START_FAILED_OTHER);
1258
1259 if (r == ELOOP) {
1260 reason = restarter_str_restarting_too_quickly;
1261 } else if (retryable) {
1262 reason = restarter_str_fault_threshold_reached;
1263 } else {
1264 reason = restarter_str_method_failed;
1265 }
1266
1267 (void) restarter_instance_update_states(local_handle, inst,
1268 RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, RERR_FAULT,
1269 reason);
1270
1271 if (!method_is_transient(inst, info->sf_method_type) &&
1272 inst->ri_i.i_primary_ctid != 0)
1273 method_remove_contract(inst, B_TRUE, B_TRUE);
1274
1275 out:
1276 inst->ri_method_thread = 0;
1277
1278 /*
1279 * Unlock the mutex after broadcasting to avoid a race condition
1280 * with restarter_delete_inst() when the 'inst' structure is freed.
1281 */
1282 (void) pthread_cond_broadcast(&inst->ri_method_cv);
1283 MUTEX_UNLOCK(&inst->ri_lock);
1284
1285 scf_instance_destroy(s_inst);
1286 scf_handle_destroy(local_handle);
1287 startd_free(info, sizeof (fork_info_t));
1288 return (NULL);
1289 }