1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /*
27 * restarter.c - service manipulation
28 *
29 * This component manages services whose restarter is svc.startd, the standard
30 * restarter. It translates restarter protocol events from the graph engine
31 * into actions on processes, as a delegated restarter would do.
32 *
33 * The master restarter manages a number of always-running threads:
34 * - restarter event thread: events from the graph engine
35 * - timeout thread: thread to fire queued timeouts
36 * - contract thread: thread to handle contract events
37 * - wait thread: thread to handle wait-based services
38 *
39 * The other threads are created as-needed:
40 * - per-instance method threads
41 * - per-instance event processing threads
42 *
43 * The interaction of all threads must result in the following conditions
44 * being satisfied (on a per-instance basis):
45 * - restarter events must be processed in order
46 * - method execution must be serialized
47 * - instance delete must be held until outstanding methods are complete
48 * - contract events shouldn't be processed while a method is running
49 * - timeouts should fire even when a method is running
50 *
51 * Service instances are represented by restarter_inst_t's and are kept in the
52 * instance_list list.
53 *
54 * Service States
55 * The current state of a service instance is kept in
56 * restarter_inst_t->ri_i.i_state. If transition to a new state could take
57 * some time, then before we effect the transition we set
58 * restarter_inst_t->ri_i.i_next_state to the target state, and afterwards we
59 * rotate i_next_state to i_state and set i_next_state to
60 * RESTARTER_STATE_NONE. So usually i_next_state is _NONE when ri_lock is not
61 * held. The exception is when we launch methods, which are done with
62 * a separate thread. To keep any other threads from grabbing ri_lock before
63 * method_thread() does, we set ri_method_thread to the thread id of the
64 * method thread, and when it is nonzero any thread with a different thread id
65 * waits on ri_method_cv.
66 *
67 * Method execution is serialized by blocking on ri_method_cv in
68 * inst_lookup_by_id() and waiting for a 0 value of ri_method_thread. This
69 * also prevents the instance structure from being deleted until all
70 * outstanding operations such as method_thread() have finished.
71 *
72 * Lock ordering:
73 *
74 * dgraph_lock [can be held when taking:]
75 * utmpx_lock
76 * dictionary->dict_lock
77 * st->st_load_lock
78 * wait_info_lock
79 * ru->restarter_update_lock
80 * restarter_queue->rpeq_lock
81 * instance_list.ril_lock
82 * inst->ri_lock
83 * st->st_configd_live_lock
84 *
85 * instance_list.ril_lock
86 * graph_queue->gpeq_lock
87 * gu->gu_lock
88 * st->st_configd_live_lock
89 * dictionary->dict_lock
90 * inst->ri_lock
91 * graph_queue->gpeq_lock
92 * gu->gu_lock
93 * tu->tu_lock
94 * tq->tq_lock
95 * inst->ri_queue_lock
96 * wait_info_lock
97 * bp->cb_lock
98 * utmpx_lock
99 *
100 * single_user_thread_lock
101 * wait_info_lock
102 * utmpx_lock
103 *
104 * gu_freeze_lock
105 *
106 * logbuf_mutex nests inside pretty much everything.
107 */
108
109 #include <sys/contract/process.h>
110 #include <sys/ctfs.h>
111 #include <sys/stat.h>
112 #include <sys/time.h>
113 #include <sys/types.h>
114 #include <sys/uio.h>
115 #include <sys/wait.h>
116 #include <assert.h>
117 #include <errno.h>
118 #include <fcntl.h>
119 #include <libcontract.h>
120 #include <libcontract_priv.h>
121 #include <libintl.h>
122 #include <librestart.h>
123 #include <librestart_priv.h>
124 #include <libuutil.h>
125 #include <limits.h>
126 #include <poll.h>
127 #include <port.h>
128 #include <pthread.h>
129 #include <stdarg.h>
130 #include <stdio.h>
131 #include <strings.h>
132 #include <unistd.h>
133
134 #include "startd.h"
135 #include "protocol.h"
136
137 static uu_list_pool_t *restarter_instance_pool;
138 static restarter_instance_list_t instance_list;
139
140 static uu_list_pool_t *restarter_queue_pool;
141
142 /*
143 * Function used to reset the restart times for an instance, when
144 * an administrative task comes along and essentially makes the times
145 * in this array ineffective.
146 */
147 static void
148 reset_start_times(restarter_inst_t *inst)
149 {
150 inst->ri_start_index = 0;
151 bzero(inst->ri_start_time, sizeof (inst->ri_start_time));
152 }
153
154 /*ARGSUSED*/
155 static int
156 restarter_instance_compare(const void *lc_arg, const void *rc_arg,
157 void *private)
158 {
159 int lc_id = ((const restarter_inst_t *)lc_arg)->ri_id;
160 int rc_id = *(int *)rc_arg;
161
162 if (lc_id > rc_id)
163 return (1);
164 if (lc_id < rc_id)
165 return (-1);
166 return (0);
167 }
168
169 static restarter_inst_t *
170 inst_lookup_by_name(const char *name)
171 {
172 int id;
173
174 id = dict_lookup_byname(name);
175 if (id == -1)
176 return (NULL);
177
178 return (inst_lookup_by_id(id));
179 }
180
181 restarter_inst_t *
182 inst_lookup_by_id(int id)
183 {
184 restarter_inst_t *inst;
185
186 MUTEX_LOCK(&instance_list.ril_lock);
187 inst = uu_list_find(instance_list.ril_instance_list, &id, NULL, NULL);
188 if (inst != NULL)
189 MUTEX_LOCK(&inst->ri_lock);
190 MUTEX_UNLOCK(&instance_list.ril_lock);
191
192 if (inst != NULL) {
193 while (inst->ri_method_thread != 0 &&
194 !pthread_equal(inst->ri_method_thread, pthread_self())) {
195 ++inst->ri_method_waiters;
196 (void) pthread_cond_wait(&inst->ri_method_cv,
197 &inst->ri_lock);
198 assert(inst->ri_method_waiters > 0);
199 --inst->ri_method_waiters;
200 }
201 }
202
203 return (inst);
204 }
205
206 static restarter_inst_t *
207 inst_lookup_queue(const char *name)
208 {
209 int id;
210 restarter_inst_t *inst;
211
212 id = dict_lookup_byname(name);
213 if (id == -1)
214 return (NULL);
215
216 MUTEX_LOCK(&instance_list.ril_lock);
217 inst = uu_list_find(instance_list.ril_instance_list, &id, NULL, NULL);
218 if (inst != NULL)
219 MUTEX_LOCK(&inst->ri_queue_lock);
220 MUTEX_UNLOCK(&instance_list.ril_lock);
221
222 return (inst);
223 }
224
225 const char *
226 service_style(int flags)
227 {
228 switch (flags & RINST_STYLE_MASK) {
229 case RINST_CONTRACT: return ("contract");
230 case RINST_TRANSIENT: return ("transient");
231 case RINST_WAIT: return ("wait");
232
233 default:
234 #ifndef NDEBUG
235 uu_warn("%s:%d: Bad flags 0x%x.\n", __FILE__, __LINE__, flags);
236 #endif
237 abort();
238 /* NOTREACHED */
239 }
240 }
241
242 /*
243 * Fails with ECONNABORTED or ECANCELED.
244 */
245 static int
246 check_contract(restarter_inst_t *inst, boolean_t primary,
247 scf_instance_t *scf_inst)
248 {
249 ctid_t *ctidp;
250 int fd, r;
251
252 ctidp = primary ? &inst->ri_i.i_primary_ctid :
253 &inst->ri_i.i_transient_ctid;
254
255 assert(*ctidp >= 1);
256
257 fd = contract_open(*ctidp, NULL, "status", O_RDONLY);
258 if (fd >= 0) {
259 r = close(fd);
260 assert(r == 0);
261 return (0);
262 }
263
264 r = restarter_remove_contract(scf_inst, *ctidp, primary ?
265 RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
266 switch (r) {
267 case 0:
268 case ECONNABORTED:
269 case ECANCELED:
270 *ctidp = 0;
271 return (r);
272
273 case ENOMEM:
274 uu_die("Out of memory\n");
275 /* NOTREACHED */
276
277 case EPERM:
278 uu_die("Insufficient privilege.\n");
279 /* NOTREACHED */
280
281 case EACCES:
282 uu_die("Repository backend access denied.\n");
283 /* NOTREACHED */
284
285 case EROFS:
286 log_error(LOG_INFO, "Could not remove unusable contract id %ld "
287 "for %s from repository.\n", *ctidp, inst->ri_i.i_fmri);
288 return (0);
289
290 case EINVAL:
291 case EBADF:
292 default:
293 assert(0);
294 abort();
295 /* NOTREACHED */
296 }
297 }
298
299 static int stop_instance(scf_handle_t *, restarter_inst_t *, stop_cause_t);
300
301 /*
302 * int restarter_insert_inst(scf_handle_t *, char *)
303 * If the inst is already in the restarter list, return its id. If the inst
304 * is not in the restarter list, initialize a restarter_inst_t, initialize its
305 * states, insert it into the list, and return 0.
306 *
307 * Fails with
308 * ENOENT - name is not in the repository
309 */
310 static int
311 restarter_insert_inst(scf_handle_t *h, const char *name)
312 {
313 int id, r;
314 restarter_inst_t *inst;
315 uu_list_index_t idx;
316 scf_service_t *scf_svc;
317 scf_instance_t *scf_inst;
318 scf_snapshot_t *snap = NULL;
319 scf_propertygroup_t *pg;
320 char *svc_name, *inst_name;
321 char logfilebuf[PATH_MAX];
322 char *c;
323 boolean_t do_commit_states;
324 restarter_instance_state_t state, next_state;
325 protocol_states_t *ps;
326 pid_t start_pid;
327 restarter_str_t reason = restarter_str_insert_in_graph;
328
329 MUTEX_LOCK(&instance_list.ril_lock);
330
331 /*
332 * We don't use inst_lookup_by_name() here because we want the lookup
333 * & insert to be atomic.
334 */
335 id = dict_lookup_byname(name);
336 if (id != -1) {
337 inst = uu_list_find(instance_list.ril_instance_list, &id, NULL,
338 &idx);
339 if (inst != NULL) {
340 MUTEX_UNLOCK(&instance_list.ril_lock);
341 return (0);
342 }
343 }
344
345 /* Allocate an instance */
346 inst = startd_zalloc(sizeof (restarter_inst_t));
347 inst->ri_utmpx_prefix = startd_alloc(max_scf_value_size);
348 inst->ri_utmpx_prefix[0] = '\0';
349
350 inst->ri_i.i_fmri = startd_alloc(strlen(name) + 1);
351 (void) strcpy((char *)inst->ri_i.i_fmri, name);
352
353 inst->ri_queue = startd_list_create(restarter_queue_pool, inst, 0);
354
355 /*
356 * id shouldn't be -1 since we use the same dictionary as graph.c, but
357 * just in case.
358 */
359 inst->ri_id = (id != -1 ? id : dict_insert(name));
360
361 special_online_hooks_get(name, &inst->ri_pre_online_hook,
362 &inst->ri_post_online_hook, &inst->ri_post_offline_hook);
363
364 scf_svc = safe_scf_service_create(h);
365 scf_inst = safe_scf_instance_create(h);
366 pg = safe_scf_pg_create(h);
367 svc_name = startd_alloc(max_scf_name_size);
368 inst_name = startd_alloc(max_scf_name_size);
369
370 rep_retry:
371 if (snap != NULL)
372 scf_snapshot_destroy(snap);
373 if (inst->ri_logstem != NULL)
374 startd_free(inst->ri_logstem, PATH_MAX);
375 if (inst->ri_common_name != NULL)
376 startd_free(inst->ri_common_name, max_scf_value_size);
377 if (inst->ri_C_common_name != NULL)
378 startd_free(inst->ri_C_common_name, max_scf_value_size);
379 snap = NULL;
380 inst->ri_logstem = NULL;
381 inst->ri_common_name = NULL;
382 inst->ri_C_common_name = NULL;
383
384 if (scf_handle_decode_fmri(h, name, NULL, scf_svc, scf_inst, NULL,
385 NULL, SCF_DECODE_FMRI_EXACT) != 0) {
386 switch (scf_error()) {
387 case SCF_ERROR_CONNECTION_BROKEN:
388 libscf_handle_rebind(h);
389 goto rep_retry;
390
391 case SCF_ERROR_NOT_FOUND:
392 goto deleted;
393 }
394
395 uu_die("Can't decode FMRI %s: %s\n", name,
396 scf_strerror(scf_error()));
397 }
398
399 /*
400 * If there's no running snapshot, then we execute using the editing
401 * snapshot. Pending snapshots will be taken later.
402 */
403 snap = libscf_get_running_snapshot(scf_inst);
404
405 if ((scf_service_get_name(scf_svc, svc_name, max_scf_name_size) < 0) ||
406 (scf_instance_get_name(scf_inst, inst_name, max_scf_name_size) <
407 0)) {
408 switch (scf_error()) {
409 case SCF_ERROR_NOT_SET:
410 break;
411
412 case SCF_ERROR_CONNECTION_BROKEN:
413 libscf_handle_rebind(h);
414 goto rep_retry;
415
416 default:
417 assert(0);
418 abort();
419 }
420
421 goto deleted;
422 }
423
424 (void) snprintf(logfilebuf, PATH_MAX, "%s:%s", svc_name, inst_name);
425 for (c = logfilebuf; *c != '\0'; c++)
426 if (*c == '/')
427 *c = '-';
428
429 inst->ri_logstem = startd_alloc(PATH_MAX);
430 (void) snprintf(inst->ri_logstem, PATH_MAX, "%s%s", logfilebuf,
431 LOG_SUFFIX);
432
433 /*
434 * If the restarter group is missing, use uninit/none. Otherwise,
435 * we're probably being restarted & don't want to mess up the states
436 * that are there.
437 */
438 state = RESTARTER_STATE_UNINIT;
439 next_state = RESTARTER_STATE_NONE;
440
441 r = scf_instance_get_pg(scf_inst, SCF_PG_RESTARTER, pg);
442 if (r != 0) {
443 switch (scf_error()) {
444 case SCF_ERROR_CONNECTION_BROKEN:
445 libscf_handle_rebind(h);
446 goto rep_retry;
447
448 case SCF_ERROR_NOT_SET:
449 goto deleted;
450
451 case SCF_ERROR_NOT_FOUND:
452 /*
453 * This shouldn't happen since the graph engine should
454 * have initialized the state to uninitialized/none if
455 * there was no restarter pg. In case somebody
456 * deleted it, though....
457 */
458 do_commit_states = B_TRUE;
459 break;
460
461 default:
462 assert(0);
463 abort();
464 }
465 } else {
466 r = libscf_read_states(pg, &state, &next_state);
467 if (r != 0) {
468 do_commit_states = B_TRUE;
469 } else {
470 if (next_state != RESTARTER_STATE_NONE) {
471 /*
472 * Force next_state to _NONE since we
473 * don't look for method processes.
474 */
475 next_state = RESTARTER_STATE_NONE;
476 do_commit_states = B_TRUE;
477 } else {
478 /*
479 * The reason for transition will depend on
480 * state.
481 */
482 if (st->st_initial == 0)
483 reason = restarter_str_startd_restart;
484 else if (state == RESTARTER_STATE_MAINT)
485 reason = restarter_str_bad_repo_state;
486 /*
487 * Inform the restarter of our state without
488 * changing the STIME in the repository.
489 */
490 ps = startd_alloc(sizeof (*ps));
491 inst->ri_i.i_state = ps->ps_state = state;
492 inst->ri_i.i_next_state = ps->ps_state_next =
493 next_state;
494 ps->ps_reason = reason;
495
496 graph_protocol_send_event(inst->ri_i.i_fmri,
497 GRAPH_UPDATE_STATE_CHANGE, ps);
498
499 do_commit_states = B_FALSE;
500 }
501 }
502 }
503
504 switch (libscf_get_startd_properties(scf_inst, snap, &inst->ri_flags,
505 &inst->ri_utmpx_prefix)) {
506 case 0:
507 break;
508
509 case ECONNABORTED:
510 libscf_handle_rebind(h);
511 goto rep_retry;
512
513 case ECANCELED:
514 goto deleted;
515
516 case ENOENT:
517 /*
518 * This is odd, because the graph engine should have required
519 * the general property group. So we'll just use default
520 * flags in anticipation of the graph engine sending us
521 * REMOVE_INSTANCE when it finds out that the general property
522 * group has been deleted.
523 */
524 inst->ri_flags = RINST_CONTRACT;
525 break;
526
527 default:
528 assert(0);
529 abort();
530 }
531
532 switch (libscf_get_template_values(scf_inst, snap,
533 &inst->ri_common_name, &inst->ri_C_common_name)) {
534 case 0:
535 break;
536
537 case ECONNABORTED:
538 libscf_handle_rebind(h);
539 goto rep_retry;
540
541 case ECANCELED:
542 goto deleted;
543
544 case ECHILD:
545 case ENOENT:
546 break;
547
548 default:
549 assert(0);
550 abort();
551 }
552
553 switch (libscf_read_method_ids(h, scf_inst, inst->ri_i.i_fmri,
554 &inst->ri_i.i_primary_ctid, &inst->ri_i.i_transient_ctid,
555 &start_pid)) {
556 case 0:
557 break;
558
559 case ECONNABORTED:
560 libscf_handle_rebind(h);
561 goto rep_retry;
562
563 case ECANCELED:
564 goto deleted;
565
566 default:
567 assert(0);
568 abort();
569 }
570
571 if (inst->ri_i.i_primary_ctid >= 1) {
572 contract_hash_store(inst->ri_i.i_primary_ctid, inst->ri_id);
573
574 switch (check_contract(inst, B_TRUE, scf_inst)) {
575 case 0:
576 break;
577
578 case ECONNABORTED:
579 libscf_handle_rebind(h);
580 goto rep_retry;
581
582 case ECANCELED:
583 goto deleted;
584
585 default:
586 assert(0);
587 abort();
588 }
589 }
590
591 if (inst->ri_i.i_transient_ctid >= 1) {
592 switch (check_contract(inst, B_FALSE, scf_inst)) {
593 case 0:
594 break;
595
596 case ECONNABORTED:
597 libscf_handle_rebind(h);
598 goto rep_retry;
599
600 case ECANCELED:
601 goto deleted;
602
603 default:
604 assert(0);
605 abort();
606 }
607 }
608
609 /* No more failures we live through, so add it to the list. */
610 (void) pthread_mutex_init(&inst->ri_lock, &mutex_attrs);
611 (void) pthread_mutex_init(&inst->ri_queue_lock, &mutex_attrs);
612 MUTEX_LOCK(&inst->ri_lock);
613 MUTEX_LOCK(&inst->ri_queue_lock);
614
615 (void) pthread_cond_init(&inst->ri_method_cv, NULL);
616
617 uu_list_node_init(inst, &inst->ri_link, restarter_instance_pool);
618 uu_list_insert(instance_list.ril_instance_list, inst, idx);
619 MUTEX_UNLOCK(&instance_list.ril_lock);
620
621 if (start_pid != -1 &&
622 (inst->ri_flags & RINST_STYLE_MASK) == RINST_WAIT) {
623 int ret;
624 ret = wait_register(start_pid, inst->ri_i.i_fmri, 0, 1);
625 if (ret == -1) {
626 /*
627 * Implication: if we can't reregister the
628 * instance, we will start another one. Two
629 * instances may or may not result in a resource
630 * conflict.
631 */
632 log_error(LOG_WARNING,
633 "%s: couldn't reregister %ld for wait\n",
634 inst->ri_i.i_fmri, start_pid);
635 } else if (ret == 1) {
636 /*
637 * Leading PID has exited.
638 */
639 (void) stop_instance(h, inst, RSTOP_EXIT);
640 }
641 }
642
643
644 scf_pg_destroy(pg);
645
646 if (do_commit_states)
647 (void) restarter_instance_update_states(h, inst, state,
648 next_state, RERR_NONE, reason);
649
650 log_framework(LOG_DEBUG, "%s is a %s-style service\n", name,
651 service_style(inst->ri_flags));
652
653 MUTEX_UNLOCK(&inst->ri_queue_lock);
654 MUTEX_UNLOCK(&inst->ri_lock);
655
656 startd_free(svc_name, max_scf_name_size);
657 startd_free(inst_name, max_scf_name_size);
658 scf_snapshot_destroy(snap);
659 scf_instance_destroy(scf_inst);
660 scf_service_destroy(scf_svc);
661
662 log_framework(LOG_DEBUG, "%s: inserted instance into restarter list\n",
663 name);
664
665 return (0);
666
667 deleted:
668 MUTEX_UNLOCK(&instance_list.ril_lock);
669 startd_free(inst_name, max_scf_name_size);
670 startd_free(svc_name, max_scf_name_size);
671 if (snap != NULL)
672 scf_snapshot_destroy(snap);
673 scf_pg_destroy(pg);
674 scf_instance_destroy(scf_inst);
675 scf_service_destroy(scf_svc);
676 startd_free((void *)inst->ri_i.i_fmri, strlen(inst->ri_i.i_fmri) + 1);
677 uu_list_destroy(inst->ri_queue);
678 if (inst->ri_logstem != NULL)
679 startd_free(inst->ri_logstem, PATH_MAX);
680 if (inst->ri_common_name != NULL)
681 startd_free(inst->ri_common_name, max_scf_value_size);
682 if (inst->ri_C_common_name != NULL)
683 startd_free(inst->ri_C_common_name, max_scf_value_size);
684 startd_free(inst->ri_utmpx_prefix, max_scf_value_size);
685 startd_free(inst, sizeof (restarter_inst_t));
686 return (ENOENT);
687 }
688
689 static void
690 restarter_delete_inst(restarter_inst_t *ri)
691 {
692 int id;
693 restarter_inst_t *rip;
694 void *cookie = NULL;
695 restarter_instance_qentry_t *e;
696
697 assert(MUTEX_HELD(&ri->ri_lock));
698
699 /*
700 * Must drop the instance lock so we can pick up the instance_list
701 * lock & remove the instance.
702 */
703 id = ri->ri_id;
704 MUTEX_UNLOCK(&ri->ri_lock);
705
706 MUTEX_LOCK(&instance_list.ril_lock);
707
708 rip = uu_list_find(instance_list.ril_instance_list, &id, NULL, NULL);
709 if (rip == NULL) {
710 MUTEX_UNLOCK(&instance_list.ril_lock);
711 return;
712 }
713
714 assert(ri == rip);
715
716 uu_list_remove(instance_list.ril_instance_list, ri);
717
718 log_framework(LOG_DEBUG, "%s: deleted instance from restarter list\n",
719 ri->ri_i.i_fmri);
720
721 MUTEX_UNLOCK(&instance_list.ril_lock);
722
723 /*
724 * We can lock the instance without holding the instance_list lock
725 * since we removed the instance from the list.
726 */
727 MUTEX_LOCK(&ri->ri_lock);
728 MUTEX_LOCK(&ri->ri_queue_lock);
729
730 if (ri->ri_i.i_primary_ctid >= 1)
731 contract_hash_remove(ri->ri_i.i_primary_ctid);
732
733 while (ri->ri_method_thread != 0 || ri->ri_method_waiters > 0)
734 (void) pthread_cond_wait(&ri->ri_method_cv, &ri->ri_lock);
735
736 while ((e = uu_list_teardown(ri->ri_queue, &cookie)) != NULL)
737 startd_free(e, sizeof (*e));
738 uu_list_destroy(ri->ri_queue);
739
740 startd_free((void *)ri->ri_i.i_fmri, strlen(ri->ri_i.i_fmri) + 1);
741 startd_free(ri->ri_logstem, PATH_MAX);
742 if (ri->ri_common_name != NULL)
743 startd_free(ri->ri_common_name, max_scf_value_size);
744 if (ri->ri_C_common_name != NULL)
745 startd_free(ri->ri_C_common_name, max_scf_value_size);
746 startd_free(ri->ri_utmpx_prefix, max_scf_value_size);
747 (void) pthread_mutex_destroy(&ri->ri_lock);
748 (void) pthread_mutex_destroy(&ri->ri_queue_lock);
749 startd_free(ri, sizeof (restarter_inst_t));
750 }
751
752 /*
753 * instance_is_wait_style()
754 *
755 * Returns 1 if the given instance is a "wait-style" service instance.
756 */
757 int
758 instance_is_wait_style(restarter_inst_t *inst)
759 {
760 assert(MUTEX_HELD(&inst->ri_lock));
761 return ((inst->ri_flags & RINST_STYLE_MASK) == RINST_WAIT);
762 }
763
764 /*
765 * instance_is_transient_style()
766 *
767 * Returns 1 if the given instance is a transient service instance.
768 */
769 int
770 instance_is_transient_style(restarter_inst_t *inst)
771 {
772 assert(MUTEX_HELD(&inst->ri_lock));
773 return ((inst->ri_flags & RINST_STYLE_MASK) == RINST_TRANSIENT);
774 }
775
776 /*
777 * instance_in_transition()
778 * Returns 1 if instance is in transition, 0 if not
779 */
780 int
781 instance_in_transition(restarter_inst_t *inst)
782 {
783 assert(MUTEX_HELD(&inst->ri_lock));
784 if (inst->ri_i.i_next_state == RESTARTER_STATE_NONE)
785 return (0);
786 return (1);
787 }
788
789 /*
790 * returns 1 if instance is already started, 0 if not
791 */
792 static int
793 instance_started(restarter_inst_t *inst)
794 {
795 int ret;
796
797 assert(MUTEX_HELD(&inst->ri_lock));
798
799 if (inst->ri_i.i_state == RESTARTER_STATE_ONLINE ||
800 inst->ri_i.i_state == RESTARTER_STATE_DEGRADED)
801 ret = 1;
802 else
803 ret = 0;
804
805 return (ret);
806 }
807
808 /*
809 * Returns
810 * 0 - success
811 * ECONNRESET - success, but h was rebound
812 */
813 int
814 restarter_instance_update_states(scf_handle_t *h, restarter_inst_t *ri,
815 restarter_instance_state_t new_state,
816 restarter_instance_state_t new_state_next, restarter_error_t err,
817 restarter_str_t reason)
818 {
819 protocol_states_t *states;
820 int e;
821 uint_t retry_count = 0, msecs = ALLOC_DELAY;
822 boolean_t rebound = B_FALSE;
823 int prev_state_online;
824 int state_online;
825
826 assert(MUTEX_HELD(&ri->ri_lock));
827
828 prev_state_online = instance_started(ri);
829
830 retry:
831 e = _restarter_commit_states(h, &ri->ri_i, new_state, new_state_next,
832 restarter_get_str_short(reason));
833 switch (e) {
834 case 0:
835 break;
836
837 case ENOMEM:
838 ++retry_count;
839 if (retry_count < ALLOC_RETRY) {
840 (void) poll(NULL, 0, msecs);
841 msecs *= ALLOC_DELAY_MULT;
842 goto retry;
843 }
844
845 /* Like startd_alloc(). */
846 uu_die("Insufficient memory.\n");
847 /* NOTREACHED */
848
849 case ECONNABORTED:
850 libscf_handle_rebind(h);
851 rebound = B_TRUE;
852 goto retry;
853
854 case EPERM:
855 case EACCES:
856 case EROFS:
857 log_error(LOG_NOTICE, "Could not commit state change for %s "
858 "to repository: %s.\n", ri->ri_i.i_fmri, strerror(e));
859 /* FALLTHROUGH */
860
861 case ENOENT:
862 ri->ri_i.i_state = new_state;
863 ri->ri_i.i_next_state = new_state_next;
864 break;
865
866 case EINVAL:
867 default:
868 bad_error("_restarter_commit_states", e);
869 }
870
871 states = startd_alloc(sizeof (protocol_states_t));
872 states->ps_state = new_state;
873 states->ps_state_next = new_state_next;
874 states->ps_err = err;
875 states->ps_reason = reason;
876 graph_protocol_send_event(ri->ri_i.i_fmri, GRAPH_UPDATE_STATE_CHANGE,
877 (void *)states);
878
879 state_online = instance_started(ri);
880
881 if (prev_state_online && !state_online)
882 ri->ri_post_offline_hook();
883 else if (!prev_state_online && state_online)
884 ri->ri_post_online_hook();
885
886 return (rebound ? ECONNRESET : 0);
887 }
888
889 void
890 restarter_mark_pending_snapshot(const char *fmri, uint_t flag)
891 {
892 restarter_inst_t *inst;
893
894 assert(flag == RINST_RETAKE_RUNNING || flag == RINST_RETAKE_START);
895
896 inst = inst_lookup_by_name(fmri);
897 if (inst == NULL)
898 return;
899
900 inst->ri_flags |= flag;
901
902 MUTEX_UNLOCK(&inst->ri_lock);
903 }
904
905 static void
906 restarter_take_pending_snapshots(scf_handle_t *h)
907 {
908 restarter_inst_t *inst;
909 int r;
910
911 MUTEX_LOCK(&instance_list.ril_lock);
912
913 for (inst = uu_list_first(instance_list.ril_instance_list);
914 inst != NULL;
915 inst = uu_list_next(instance_list.ril_instance_list, inst)) {
916 const char *fmri;
917 scf_instance_t *sinst = NULL;
918
919 MUTEX_LOCK(&inst->ri_lock);
920
921 /*
922 * This is where we'd check inst->ri_method_thread and if it
923 * were nonzero we'd wait in anticipation of another thread
924 * executing a method for inst. Doing so with the instance_list
925 * locked, though, leads to deadlock. Since taking a snapshot
926 * during that window won't hurt anything, we'll just continue.
927 */
928
929 fmri = inst->ri_i.i_fmri;
930
931 if (inst->ri_flags & RINST_RETAKE_RUNNING) {
932 scf_snapshot_t *rsnap;
933
934 (void) libscf_fmri_get_instance(h, fmri, &sinst);
935
936 rsnap = libscf_get_or_make_running_snapshot(sinst,
937 fmri, B_FALSE);
938
939 scf_instance_destroy(sinst);
940
941 if (rsnap != NULL)
942 inst->ri_flags &= ~RINST_RETAKE_RUNNING;
943
944 scf_snapshot_destroy(rsnap);
945 }
946
947 if (inst->ri_flags & RINST_RETAKE_START) {
948 switch (r = libscf_snapshots_poststart(h, fmri,
949 B_FALSE)) {
950 case 0:
951 case ENOENT:
952 inst->ri_flags &= ~RINST_RETAKE_START;
953 break;
954
955 case ECONNABORTED:
956 break;
957
958 case EACCES:
959 default:
960 bad_error("libscf_snapshots_poststart", r);
961 }
962 }
963
964 MUTEX_UNLOCK(&inst->ri_lock);
965 }
966
967 MUTEX_UNLOCK(&instance_list.ril_lock);
968 }
969
970 /* ARGSUSED */
971 void *
972 restarter_post_fsminimal_thread(void *unused)
973 {
974 scf_handle_t *h;
975 int r;
976
977 h = libscf_handle_create_bound_loop();
978
979 for (;;) {
980 r = libscf_create_self(h);
981 if (r == 0)
982 break;
983
984 assert(r == ECONNABORTED);
985 libscf_handle_rebind(h);
986 }
987
988 restarter_take_pending_snapshots(h);
989
990 (void) scf_handle_unbind(h);
991 scf_handle_destroy(h);
992
993 return (NULL);
994 }
995
996 /*
997 * int stop_instance()
998 *
999 * Stop the instance identified by the instance given as the second argument,
1000 * for the cause stated.
1001 *
1002 * Returns
1003 * 0 - success
1004 * -1 - inst is in transition
1005 */
1006 static int
1007 stop_instance(scf_handle_t *local_handle, restarter_inst_t *inst,
1008 stop_cause_t cause)
1009 {
1010 fork_info_t *info;
1011 const char *cp;
1012 int err;
1013 restarter_error_t re;
1014 restarter_str_t reason;
1015
1016 assert(MUTEX_HELD(&inst->ri_lock));
1017 assert(inst->ri_method_thread == 0);
1018
1019 switch (cause) {
1020 case RSTOP_EXIT:
1021 re = RERR_RESTART;
1022 reason = restarter_str_ct_ev_exit;
1023 cp = "all processes in service exited";
1024 break;
1025 case RSTOP_CORE:
1026 re = RERR_FAULT;
1027 reason = restarter_str_ct_ev_core;
1028 cp = "process dumped core";
1029 break;
1030 case RSTOP_SIGNAL:
1031 re = RERR_FAULT;
1032 reason = restarter_str_ct_ev_signal;
1033 cp = "process received fatal signal from outside the service";
1034 break;
1035 case RSTOP_HWERR:
1036 re = RERR_FAULT;
1037 reason = restarter_str_ct_ev_hwerr;
1038 cp = "process killed due to uncorrectable hardware error";
1039 break;
1040 case RSTOP_DEPENDENCY:
1041 re = RERR_RESTART;
1042 reason = restarter_str_dependency_activity;
1043 cp = "dependency activity requires stop";
1044 break;
1045 case RSTOP_DISABLE:
1046 re = RERR_RESTART;
1047 reason = restarter_str_disable_request;
1048 cp = "service disabled";
1049 break;
1050 case RSTOP_RESTART:
1051 re = RERR_RESTART;
1052 reason = restarter_str_restart_request;
1053 cp = "service restarting";
1054 break;
1055 default:
1056 #ifndef NDEBUG
1057 (void) fprintf(stderr, "Unknown cause %d at %s:%d.\n",
1058 cause, __FILE__, __LINE__);
1059 #endif
1060 abort();
1061 }
1062
1063 /* Services in the disabled and maintenance state are ignored */
1064 if (inst->ri_i.i_state == RESTARTER_STATE_MAINT ||
1065 inst->ri_i.i_state == RESTARTER_STATE_DISABLED) {
1066 log_framework(LOG_DEBUG,
1067 "%s: stop_instance -> is maint/disabled\n",
1068 inst->ri_i.i_fmri);
1069 return (0);
1070 }
1071
1072 /* Already stopped instances are left alone */
1073 if (instance_started(inst) == 0) {
1074 log_framework(LOG_DEBUG, "Restarter: %s is already stopped.\n",
1075 inst->ri_i.i_fmri);
1076 return (0);
1077 }
1078
1079 if (instance_in_transition(inst)) {
1080 /* requeue event by returning -1 */
1081 log_framework(LOG_DEBUG,
1082 "Restarter: Not stopping %s, in transition.\n",
1083 inst->ri_i.i_fmri);
1084 return (-1);
1085 }
1086
1087 log_instance(inst, B_TRUE, "Stopping because %s.", cp);
1088
1089 log_framework(re == RERR_FAULT ? LOG_INFO : LOG_DEBUG,
1090 "%s: Instance stopping because %s.\n", inst->ri_i.i_fmri, cp);
1091
1092 if (instance_is_wait_style(inst) && cause == RSTOP_EXIT) {
1093 /*
1094 * No need to stop instance, as child has exited; remove
1095 * contract and move the instance to the offline state.
1096 */
1097 switch (err = restarter_instance_update_states(local_handle,
1098 inst, inst->ri_i.i_state, RESTARTER_STATE_OFFLINE, re,
1099 reason)) {
1100 case 0:
1101 case ECONNRESET:
1102 break;
1103
1104 default:
1105 bad_error("restarter_instance_update_states", err);
1106 }
1107
1108 (void) update_fault_count(inst, FAULT_COUNT_RESET);
1109 reset_start_times(inst);
1110
1111 if (inst->ri_i.i_primary_ctid != 0) {
1112 inst->ri_m_inst =
1113 safe_scf_instance_create(local_handle);
1114 inst->ri_mi_deleted = B_FALSE;
1115
1116 libscf_reget_instance(inst);
1117 method_remove_contract(inst, B_TRUE, B_TRUE);
1118
1119 scf_instance_destroy(inst->ri_m_inst);
1120 inst->ri_m_inst = NULL;
1121 }
1122
1123 switch (err = restarter_instance_update_states(local_handle,
1124 inst, inst->ri_i.i_next_state, RESTARTER_STATE_NONE, re,
1125 reason)) {
1126 case 0:
1127 case ECONNRESET:
1128 break;
1129
1130 default:
1131 bad_error("restarter_instance_update_states", err);
1132 }
1133
1134 return (0);
1135 } else if (instance_is_wait_style(inst) && re == RERR_RESTART) {
1136 /*
1137 * Stopping a wait service through means other than the pid
1138 * exiting should keep wait_thread() from restarting the
1139 * service, by removing it from the wait list.
1140 * We cannot remove it right now otherwise the process will
1141 * end up <defunct> so mark it to be ignored.
1142 */
1143 wait_ignore_by_fmri(inst->ri_i.i_fmri);
1144 }
1145
1146 switch (err = restarter_instance_update_states(local_handle, inst,
1147 inst->ri_i.i_state, inst->ri_i.i_enabled ? RESTARTER_STATE_OFFLINE :
1148 RESTARTER_STATE_DISABLED, RERR_NONE, reason)) {
1149 case 0:
1150 case ECONNRESET:
1151 break;
1152
1153 default:
1154 bad_error("restarter_instance_update_states", err);
1155 }
1156
1157 info = startd_zalloc(sizeof (fork_info_t));
1158
1159 info->sf_id = inst->ri_id;
1160 info->sf_method_type = METHOD_STOP;
1161 info->sf_event_type = re;
1162 info->sf_reason = reason;
1163 inst->ri_method_thread = startd_thread_create(method_thread, info);
1164
1165 return (0);
1166 }
1167
1168 /*
1169 * Returns
1170 * ENOENT - fmri is not in instance_list
1171 * 0 - success
1172 * ECONNRESET - success, though handle was rebound
1173 * -1 - instance is in transition
1174 */
1175 int
1176 stop_instance_fmri(scf_handle_t *h, const char *fmri, uint_t flags)
1177 {
1178 restarter_inst_t *rip;
1179 int r;
1180
1181 rip = inst_lookup_by_name(fmri);
1182 if (rip == NULL)
1183 return (ENOENT);
1184
1185 r = stop_instance(h, rip, flags);
1186
1187 MUTEX_UNLOCK(&rip->ri_lock);
1188
1189 return (r);
1190 }
1191
1192 static void
1193 unmaintain_instance(scf_handle_t *h, restarter_inst_t *rip,
1194 unmaint_cause_t cause)
1195 {
1196 ctid_t ctid;
1197 scf_instance_t *inst;
1198 int r;
1199 uint_t tries = 0, msecs = ALLOC_DELAY;
1200 const char *cp;
1201 restarter_str_t reason;
1202
1203 assert(MUTEX_HELD(&rip->ri_lock));
1204
1205 if (rip->ri_i.i_state != RESTARTER_STATE_MAINT) {
1206 log_error(LOG_DEBUG, "Restarter: "
1207 "Ignoring maintenance off command because %s is not in the "
1208 "maintenance state.\n", rip->ri_i.i_fmri);
1209 return;
1210 }
1211
1212 switch (cause) {
1213 case RUNMAINT_CLEAR:
1214 cp = "clear requested";
1215 reason = restarter_str_clear_request;
1216 break;
1217 case RUNMAINT_DISABLE:
1218 cp = "disable requested";
1219 reason = restarter_str_disable_request;
1220 break;
1221 default:
1222 #ifndef NDEBUG
1223 (void) fprintf(stderr, "Uncaught case for %d at %s:%d.\n",
1224 cause, __FILE__, __LINE__);
1225 #endif
1226 abort();
1227 }
1228
1229 log_instance(rip, B_TRUE, "Leaving maintenance because %s.",
1230 cp);
1231 log_framework(LOG_DEBUG, "%s: Instance leaving maintenance because "
1232 "%s.\n", rip->ri_i.i_fmri, cp);
1233
1234 (void) restarter_instance_update_states(h, rip, RESTARTER_STATE_UNINIT,
1235 RESTARTER_STATE_NONE, RERR_RESTART, reason);
1236
1237 /*
1238 * If we did ADMIN_MAINT_ON_IMMEDIATE, then there might still be
1239 * a primary contract.
1240 */
1241 if (rip->ri_i.i_primary_ctid == 0)
1242 return;
1243
1244 ctid = rip->ri_i.i_primary_ctid;
1245 contract_abandon(ctid);
1246 rip->ri_i.i_primary_ctid = 0;
1247
1248 rep_retry:
1249 switch (r = libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst)) {
1250 case 0:
1251 break;
1252
1253 case ECONNABORTED:
1254 libscf_handle_rebind(h);
1255 goto rep_retry;
1256
1257 case ENOENT:
1258 /* Must have been deleted. */
1259 return;
1260
1261 case EINVAL:
1262 case ENOTSUP:
1263 default:
1264 bad_error("libscf_handle_rebind", r);
1265 }
1266
1267 again:
1268 r = restarter_remove_contract(inst, ctid, RESTARTER_CONTRACT_PRIMARY);
1269 switch (r) {
1270 case 0:
1271 break;
1272
1273 case ENOMEM:
1274 ++tries;
1275 if (tries < ALLOC_RETRY) {
1276 (void) poll(NULL, 0, msecs);
1277 msecs *= ALLOC_DELAY_MULT;
1278 goto again;
1279 }
1280
1281 uu_die("Insufficient memory.\n");
1282 /* NOTREACHED */
1283
1284 case ECONNABORTED:
1285 scf_instance_destroy(inst);
1286 libscf_handle_rebind(h);
1287 goto rep_retry;
1288
1289 case ECANCELED:
1290 break;
1291
1292 case EPERM:
1293 case EACCES:
1294 case EROFS:
1295 log_error(LOG_INFO,
1296 "Could not remove contract id %lu for %s (%s).\n", ctid,
1297 rip->ri_i.i_fmri, strerror(r));
1298 break;
1299
1300 case EINVAL:
1301 case EBADF:
1302 default:
1303 bad_error("restarter_remove_contract", r);
1304 }
1305
1306 scf_instance_destroy(inst);
1307 }
1308
1309 /*
1310 * enable_inst()
1311 * Set inst->ri_i.i_enabled. Expects 'e' to be _ENABLE, _DISABLE, or
1312 * _ADMIN_DISABLE. If the event is _ENABLE and inst is uninitialized or
1313 * disabled, move it to offline. If the event is _DISABLE or
1314 * _ADMIN_DISABLE, make sure inst will move to disabled.
1315 *
1316 * Returns
1317 * 0 - success
1318 * ECONNRESET - h was rebound
1319 */
1320 static int
1321 enable_inst(scf_handle_t *h, restarter_inst_t *inst,
1322 restarter_instance_qentry_t *riq)
1323 {
1324 restarter_instance_state_t state;
1325 restarter_event_type_t e = riq->riq_type;
1326 restarter_str_t reason = restarter_str_per_configuration;
1327 int r;
1328
1329 assert(MUTEX_HELD(&inst->ri_lock));
1330 assert(e == RESTARTER_EVENT_TYPE_ADMIN_DISABLE ||
1331 e == RESTARTER_EVENT_TYPE_DISABLE ||
1332 e == RESTARTER_EVENT_TYPE_ENABLE);
1333 assert(instance_in_transition(inst) == 0);
1334
1335 state = inst->ri_i.i_state;
1336
1337 if (e == RESTARTER_EVENT_TYPE_ENABLE) {
1338 inst->ri_i.i_enabled = 1;
1339
1340 if (state == RESTARTER_STATE_UNINIT ||
1341 state == RESTARTER_STATE_DISABLED) {
1342 /*
1343 * B_FALSE: Don't log an error if the log_instance()
1344 * fails because it will fail on the miniroot before
1345 * install-discovery runs.
1346 */
1347 log_instance(inst, B_FALSE, "Enabled.");
1348 log_framework(LOG_DEBUG, "%s: Instance enabled.\n",
1349 inst->ri_i.i_fmri);
1350
1351 /*
1352 * If we are coming from DISABLED, it was obviously an
1353 * enable request. If we are coming from UNINIT, it may
1354 * have been a sevice in MAINT that was cleared.
1355 */
1356 if (riq->riq_reason == restarter_str_clear_request)
1357 reason = restarter_str_clear_request;
1358 else if (state == RESTARTER_STATE_DISABLED)
1359 reason = restarter_str_enable_request;
1360 (void) restarter_instance_update_states(h, inst,
1361 RESTARTER_STATE_OFFLINE, RESTARTER_STATE_NONE,
1362 RERR_NONE, reason);
1363 } else {
1364 log_framework(LOG_DEBUG, "Restarter: "
1365 "Not changing state of %s for enable command.\n",
1366 inst->ri_i.i_fmri);
1367 }
1368 } else {
1369 inst->ri_i.i_enabled = 0;
1370
1371 switch (state) {
1372 case RESTARTER_STATE_ONLINE:
1373 case RESTARTER_STATE_DEGRADED:
1374 r = stop_instance(h, inst, RSTOP_DISABLE);
1375 return (r == ECONNRESET ? 0 : r);
1376
1377 case RESTARTER_STATE_OFFLINE:
1378 case RESTARTER_STATE_UNINIT:
1379 if (inst->ri_i.i_primary_ctid != 0) {
1380 inst->ri_m_inst = safe_scf_instance_create(h);
1381 inst->ri_mi_deleted = B_FALSE;
1382
1383 libscf_reget_instance(inst);
1384 method_remove_contract(inst, B_TRUE, B_TRUE);
1385
1386 scf_instance_destroy(inst->ri_m_inst);
1387 }
1388 /* B_FALSE: See log_instance(..., "Enabled."); above */
1389 log_instance(inst, B_FALSE, "Disabled.");
1390 log_framework(LOG_DEBUG, "%s: Instance disabled.\n",
1391 inst->ri_i.i_fmri);
1392
1393 /*
1394 * If we are coming from OFFLINE, it was obviously a
1395 * disable request. But if we are coming from
1396 * UNINIT, it may have been a disable request for a
1397 * service in MAINT.
1398 */
1399 if (riq->riq_reason == restarter_str_disable_request ||
1400 state == RESTARTER_STATE_OFFLINE)
1401 reason = restarter_str_disable_request;
1402 (void) restarter_instance_update_states(h, inst,
1403 RESTARTER_STATE_DISABLED, RESTARTER_STATE_NONE,
1404 RERR_RESTART, reason);
1405 return (0);
1406
1407 case RESTARTER_STATE_DISABLED:
1408 break;
1409
1410 case RESTARTER_STATE_MAINT:
1411 /*
1412 * We only want to pull the instance out of maintenance
1413 * if the disable is on adminstrative request. The
1414 * graph engine sends _DISABLE events whenever a
1415 * service isn't in the disabled state, and we don't
1416 * want to pull the service out of maintenance if,
1417 * for example, it is there due to a dependency cycle.
1418 */
1419 if (e == RESTARTER_EVENT_TYPE_ADMIN_DISABLE)
1420 unmaintain_instance(h, inst, RUNMAINT_DISABLE);
1421 break;
1422
1423 default:
1424 #ifndef NDEBUG
1425 (void) fprintf(stderr, "Restarter instance %s has "
1426 "unknown state %d.\n", inst->ri_i.i_fmri, state);
1427 #endif
1428 abort();
1429 }
1430 }
1431
1432 return (0);
1433 }
1434
1435 static void
1436 start_instance(scf_handle_t *local_handle, restarter_inst_t *inst,
1437 int32_t reason)
1438 {
1439 fork_info_t *info;
1440 restarter_str_t new_reason;
1441
1442 assert(MUTEX_HELD(&inst->ri_lock));
1443 assert(instance_in_transition(inst) == 0);
1444 assert(inst->ri_method_thread == 0);
1445
1446 log_framework(LOG_DEBUG, "%s: trying to start instance\n",
1447 inst->ri_i.i_fmri);
1448
1449 /*
1450 * We want to keep the original reason for restarts and clear actions
1451 */
1452 switch (reason) {
1453 case restarter_str_restart_request:
1454 case restarter_str_clear_request:
1455 new_reason = reason;
1456 break;
1457 default:
1458 new_reason = restarter_str_dependencies_satisfied;
1459 }
1460
1461 /* Services in the disabled and maintenance state are ignored */
1462 if (inst->ri_i.i_state == RESTARTER_STATE_MAINT ||
1463 inst->ri_i.i_state == RESTARTER_STATE_DISABLED ||
1464 inst->ri_i.i_enabled == 0) {
1465 log_framework(LOG_DEBUG,
1466 "%s: start_instance -> is maint/disabled\n",
1467 inst->ri_i.i_fmri);
1468 return;
1469 }
1470
1471 /* Already started instances are left alone */
1472 if (instance_started(inst) == 1) {
1473 log_framework(LOG_DEBUG,
1474 "%s: start_instance -> is already started\n",
1475 inst->ri_i.i_fmri);
1476 return;
1477 }
1478
1479 log_framework(LOG_DEBUG, "%s: starting instance.\n", inst->ri_i.i_fmri);
1480
1481 (void) restarter_instance_update_states(local_handle, inst,
1482 inst->ri_i.i_state, RESTARTER_STATE_ONLINE, RERR_NONE, new_reason);
1483
1484 info = startd_zalloc(sizeof (fork_info_t));
1485
1486 info->sf_id = inst->ri_id;
1487 info->sf_method_type = METHOD_START;
1488 info->sf_event_type = RERR_NONE;
1489 info->sf_reason = new_reason;
1490 inst->ri_method_thread = startd_thread_create(method_thread, info);
1491 }
1492
1493 static int
1494 event_from_tty(scf_handle_t *h, restarter_inst_t *rip)
1495 {
1496 scf_instance_t *inst;
1497 int ret = 0;
1498
1499 if (libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst))
1500 return (-1);
1501
1502 ret = restarter_inst_ractions_from_tty(inst);
1503
1504 scf_instance_destroy(inst);
1505 return (ret);
1506 }
1507
1508 static void
1509 maintain_instance(scf_handle_t *h, restarter_inst_t *rip, int immediate,
1510 restarter_str_t reason)
1511 {
1512 fork_info_t *info;
1513 scf_instance_t *scf_inst = NULL;
1514
1515 assert(MUTEX_HELD(&rip->ri_lock));
1516 assert(reason != restarter_str_none);
1517 assert(rip->ri_method_thread == 0);
1518
1519 log_instance(rip, B_TRUE, "Stopping for maintenance due to %s.",
1520 restarter_get_str_short(reason));
1521 log_framework(LOG_DEBUG, "%s: stopping for maintenance due to %s.\n",
1522 rip->ri_i.i_fmri, restarter_get_str_short(reason));
1523
1524 /* Services in the maintenance state are ignored */
1525 if (rip->ri_i.i_state == RESTARTER_STATE_MAINT) {
1526 log_framework(LOG_DEBUG,
1527 "%s: maintain_instance -> is already in maintenance\n",
1528 rip->ri_i.i_fmri);
1529 return;
1530 }
1531
1532 /*
1533 * If reason state is restarter_str_service_request and
1534 * restarter_actions/auxiliary_fmri property is set with a valid fmri,
1535 * copy the fmri to restarter/auxiliary_fmri so svcs -x can use.
1536 */
1537 if (reason == restarter_str_service_request &&
1538 libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &scf_inst) == 0) {
1539 if (restarter_inst_validate_ractions_aux_fmri(scf_inst) == 0) {
1540 if (restarter_inst_set_aux_fmri(scf_inst))
1541 log_framework(LOG_DEBUG, "%s: "
1542 "restarter_inst_set_aux_fmri failed: ",
1543 rip->ri_i.i_fmri);
1544 } else {
1545 log_framework(LOG_DEBUG, "%s: "
1546 "restarter_inst_validate_ractions_aux_fmri "
1547 "failed: ", rip->ri_i.i_fmri);
1548
1549 if (restarter_inst_reset_aux_fmri(scf_inst))
1550 log_framework(LOG_DEBUG, "%s: "
1551 "restarter_inst_reset_aux_fmri failed: ",
1552 rip->ri_i.i_fmri);
1553 }
1554 scf_instance_destroy(scf_inst);
1555 }
1556
1557 if (immediate || !instance_started(rip)) {
1558 if (rip->ri_i.i_primary_ctid != 0) {
1559 rip->ri_m_inst = safe_scf_instance_create(h);
1560 rip->ri_mi_deleted = B_FALSE;
1561
1562 libscf_reget_instance(rip);
1563 method_remove_contract(rip, B_TRUE, B_TRUE);
1564
1565 scf_instance_destroy(rip->ri_m_inst);
1566 }
1567
1568 (void) restarter_instance_update_states(h, rip,
1569 RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, RERR_RESTART,
1570 reason);
1571 return;
1572 }
1573
1574 (void) restarter_instance_update_states(h, rip, rip->ri_i.i_state,
1575 RESTARTER_STATE_MAINT, RERR_NONE, reason);
1576
1577 log_transition(rip, MAINT_REQUESTED);
1578
1579 info = startd_zalloc(sizeof (*info));
1580 info->sf_id = rip->ri_id;
1581 info->sf_method_type = METHOD_STOP;
1582 info->sf_event_type = RERR_RESTART;
1583 info->sf_reason = reason;
1584 rip->ri_method_thread = startd_thread_create(method_thread, info);
1585 }
1586
1587 static void
1588 refresh_instance(scf_handle_t *h, restarter_inst_t *rip)
1589 {
1590 scf_instance_t *inst;
1591 scf_snapshot_t *snap;
1592 fork_info_t *info;
1593 int r;
1594
1595 assert(MUTEX_HELD(&rip->ri_lock));
1596
1597 log_instance(rip, B_TRUE, "Rereading configuration.");
1598 log_framework(LOG_DEBUG, "%s: rereading configuration.\n",
1599 rip->ri_i.i_fmri);
1600
1601 rep_retry:
1602 r = libscf_fmri_get_instance(h, rip->ri_i.i_fmri, &inst);
1603 switch (r) {
1604 case 0:
1605 break;
1606
1607 case ECONNABORTED:
1608 libscf_handle_rebind(h);
1609 goto rep_retry;
1610
1611 case ENOENT:
1612 /* Must have been deleted. */
1613 return;
1614
1615 case EINVAL:
1616 case ENOTSUP:
1617 default:
1618 bad_error("libscf_fmri_get_instance", r);
1619 }
1620
1621 snap = libscf_get_running_snapshot(inst);
1622
1623 r = libscf_get_startd_properties(inst, snap, &rip->ri_flags,
1624 &rip->ri_utmpx_prefix);
1625 switch (r) {
1626 case 0:
1627 log_framework(LOG_DEBUG, "%s is a %s-style service\n",
1628 rip->ri_i.i_fmri, service_style(rip->ri_flags));
1629 break;
1630
1631 case ECONNABORTED:
1632 scf_instance_destroy(inst);
1633 scf_snapshot_destroy(snap);
1634 libscf_handle_rebind(h);
1635 goto rep_retry;
1636
1637 case ECANCELED:
1638 case ENOENT:
1639 /* Succeed in anticipation of REMOVE_INSTANCE. */
1640 break;
1641
1642 default:
1643 bad_error("libscf_get_startd_properties", r);
1644 }
1645
1646 if (instance_started(rip)) {
1647 /* Refresh does not change the state. */
1648 (void) restarter_instance_update_states(h, rip,
1649 rip->ri_i.i_state, rip->ri_i.i_state, RERR_NONE,
1650 restarter_str_refresh);
1651
1652 info = startd_zalloc(sizeof (*info));
1653 info->sf_id = rip->ri_id;
1654 info->sf_method_type = METHOD_REFRESH;
1655 info->sf_event_type = RERR_REFRESH;
1656 info->sf_reason = NULL;
1657
1658 assert(rip->ri_method_thread == 0);
1659 rip->ri_method_thread =
1660 startd_thread_create(method_thread, info);
1661 }
1662
1663 scf_snapshot_destroy(snap);
1664 scf_instance_destroy(inst);
1665 }
1666
1667 const char *event_names[] = { "INVALID", "ADD_INSTANCE", "REMOVE_INSTANCE",
1668 "ENABLE", "DISABLE", "ADMIN_DEGRADED", "ADMIN_REFRESH",
1669 "ADMIN_RESTART", "ADMIN_MAINT_OFF", "ADMIN_MAINT_ON",
1670 "ADMIN_MAINT_ON_IMMEDIATE", "STOP", "START", "DEPENDENCY_CYCLE",
1671 "INVALID_DEPENDENCY", "ADMIN_DISABLE", "STOP_RESET"
1672 };
1673
1674 /*
1675 * void *restarter_process_events()
1676 *
1677 * Called in a separate thread to process the events on an instance's
1678 * queue. Empties the queue completely, and tries to keep the thread
1679 * around for a little while after the queue is empty to save on
1680 * startup costs.
1681 */
1682 static void *
1683 restarter_process_events(void *arg)
1684 {
1685 scf_handle_t *h;
1686 restarter_instance_qentry_t *event;
1687 restarter_inst_t *rip;
1688 char *fmri = (char *)arg;
1689 struct timespec to;
1690
1691 assert(fmri != NULL);
1692
1693 h = libscf_handle_create_bound_loop();
1694
1695 /* grab the queue lock */
1696 rip = inst_lookup_queue(fmri);
1697 if (rip == NULL)
1698 goto out;
1699
1700 again:
1701
1702 while ((event = uu_list_first(rip->ri_queue)) != NULL) {
1703 restarter_inst_t *inst;
1704
1705 /* drop the queue lock */
1706 MUTEX_UNLOCK(&rip->ri_queue_lock);
1707
1708 /*
1709 * Grab the inst lock -- this waits until any outstanding
1710 * method finishes running.
1711 */
1712 inst = inst_lookup_by_name(fmri);
1713 if (inst == NULL) {
1714 /* Getting deleted in the middle isn't an error. */
1715 goto cont;
1716 }
1717
1718 assert(instance_in_transition(inst) == 0);
1719
1720 /* process the event */
1721 switch (event->riq_type) {
1722 case RESTARTER_EVENT_TYPE_ENABLE:
1723 case RESTARTER_EVENT_TYPE_DISABLE:
1724 (void) enable_inst(h, inst, event);
1725 break;
1726
1727 case RESTARTER_EVENT_TYPE_ADMIN_DISABLE:
1728 if (enable_inst(h, inst, event) == 0)
1729 reset_start_times(inst);
1730 break;
1731
1732 case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE:
1733 restarter_delete_inst(inst);
1734 inst = NULL;
1735 goto cont;
1736
1737 case RESTARTER_EVENT_TYPE_STOP_RESET:
1738 reset_start_times(inst);
1739 /* FALLTHROUGH */
1740 case RESTARTER_EVENT_TYPE_STOP:
1741 (void) stop_instance(h, inst, RSTOP_DEPENDENCY);
1742 break;
1743
1744 case RESTARTER_EVENT_TYPE_START:
1745 start_instance(h, inst, event->riq_reason);
1746 break;
1747
1748 case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE:
1749 maintain_instance(h, inst, 0,
1750 restarter_str_dependency_cycle);
1751 break;
1752
1753 case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY:
1754 maintain_instance(h, inst, 0,
1755 restarter_str_invalid_dependency);
1756 break;
1757
1758 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
1759 if (event_from_tty(h, inst) == 0)
1760 maintain_instance(h, inst, 0,
1761 restarter_str_service_request);
1762 else
1763 maintain_instance(h, inst, 0,
1764 restarter_str_administrative_request);
1765 break;
1766
1767 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
1768 if (event_from_tty(h, inst) == 0)
1769 maintain_instance(h, inst, 1,
1770 restarter_str_service_request);
1771 else
1772 maintain_instance(h, inst, 1,
1773 restarter_str_administrative_request);
1774 break;
1775
1776 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
1777 unmaintain_instance(h, inst, RUNMAINT_CLEAR);
1778 reset_start_times(inst);
1779 break;
1780
1781 case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
1782 refresh_instance(h, inst);
1783 break;
1784
1785 case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
1786 log_framework(LOG_WARNING, "Restarter: "
1787 "%s command (for %s) unimplemented.\n",
1788 event_names[event->riq_type], inst->ri_i.i_fmri);
1789 break;
1790
1791 case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
1792 if (!instance_started(inst)) {
1793 log_framework(LOG_DEBUG, "Restarter: "
1794 "Not restarting %s; not running.\n",
1795 inst->ri_i.i_fmri);
1796 } else {
1797 /*
1798 * Stop the instance. If it can be restarted,
1799 * the graph engine will send a new event.
1800 */
1801 if (stop_instance(h, inst, RSTOP_RESTART) == 0)
1802 reset_start_times(inst);
1803 }
1804 break;
1805
1806 case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
1807 default:
1808 #ifndef NDEBUG
1809 uu_warn("%s:%d: Bad restarter event %d. "
1810 "Aborting.\n", __FILE__, __LINE__, event->riq_type);
1811 #endif
1812 abort();
1813 }
1814
1815 assert(inst != NULL);
1816 MUTEX_UNLOCK(&inst->ri_lock);
1817
1818 cont:
1819 /* grab the queue lock */
1820 rip = inst_lookup_queue(fmri);
1821 if (rip == NULL)
1822 goto out;
1823
1824 /* delete the event */
1825 uu_list_remove(rip->ri_queue, event);
1826 startd_free(event, sizeof (restarter_instance_qentry_t));
1827 }
1828
1829 assert(rip != NULL);
1830
1831 /*
1832 * Try to preserve the thread for a little while for future use.
1833 */
1834 to.tv_sec = 3;
1835 to.tv_nsec = 0;
1836 (void) pthread_cond_reltimedwait_np(&rip->ri_queue_cv,
1837 &rip->ri_queue_lock, &to);
1838
1839 if (uu_list_first(rip->ri_queue) != NULL)
1840 goto again;
1841
1842 rip->ri_queue_thread = 0;
1843 MUTEX_UNLOCK(&rip->ri_queue_lock);
1844 out:
1845 (void) scf_handle_unbind(h);
1846 scf_handle_destroy(h);
1847 free(fmri);
1848 return (NULL);
1849 }
1850
1851 static int
1852 is_admin_event(restarter_event_type_t t) {
1853
1854 switch (t) {
1855 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON:
1856 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE:
1857 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF:
1858 case RESTARTER_EVENT_TYPE_ADMIN_REFRESH:
1859 case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED:
1860 case RESTARTER_EVENT_TYPE_ADMIN_RESTART:
1861 return (1);
1862 default:
1863 return (0);
1864 }
1865 }
1866
1867 static void
1868 restarter_queue_event(restarter_inst_t *ri, restarter_protocol_event_t *e)
1869 {
1870 restarter_instance_qentry_t *qe;
1871 int r;
1872
1873 assert(MUTEX_HELD(&ri->ri_queue_lock));
1874 assert(!MUTEX_HELD(&ri->ri_lock));
1875
1876 qe = startd_zalloc(sizeof (restarter_instance_qentry_t));
1877 qe->riq_type = e->rpe_type;
1878 qe->riq_reason = e->rpe_reason;
1879
1880 uu_list_node_init(qe, &qe->riq_link, restarter_queue_pool);
1881 r = uu_list_insert_before(ri->ri_queue, NULL, qe);
1882 assert(r == 0);
1883 }
1884
1885 /*
1886 * void *restarter_event_thread()
1887 *
1888 * Handle incoming graph events by placing them on a per-instance
1889 * queue. We can't lock the main part of the instance structure, so
1890 * just modify the seprarately locked event queue portion.
1891 */
1892 /*ARGSUSED*/
1893 static void *
1894 restarter_event_thread(void *unused)
1895 {
1896 scf_handle_t *h;
1897
1898 /*
1899 * This is a new thread, and thus, gets its own handle
1900 * to the repository.
1901 */
1902 h = libscf_handle_create_bound_loop();
1903
1904 MUTEX_LOCK(&ru->restarter_update_lock);
1905
1906 /*CONSTCOND*/
1907 while (1) {
1908 restarter_protocol_event_t *e;
1909
1910 while (ru->restarter_update_wakeup == 0)
1911 (void) pthread_cond_wait(&ru->restarter_update_cv,
1912 &ru->restarter_update_lock);
1913
1914 ru->restarter_update_wakeup = 0;
1915
1916 while ((e = restarter_event_dequeue()) != NULL) {
1917 restarter_inst_t *rip;
1918 char *fmri;
1919
1920 MUTEX_UNLOCK(&ru->restarter_update_lock);
1921
1922 /*
1923 * ADD_INSTANCE is special: there's likely no
1924 * instance structure yet, so we need to handle the
1925 * addition synchronously.
1926 */
1927 switch (e->rpe_type) {
1928 case RESTARTER_EVENT_TYPE_ADD_INSTANCE:
1929 if (restarter_insert_inst(h, e->rpe_inst) != 0)
1930 log_error(LOG_INFO, "Restarter: "
1931 "Could not add %s.\n", e->rpe_inst);
1932
1933 MUTEX_LOCK(&st->st_load_lock);
1934 if (--st->st_load_instances == 0)
1935 (void) pthread_cond_broadcast(
1936 &st->st_load_cv);
1937 MUTEX_UNLOCK(&st->st_load_lock);
1938
1939 goto nolookup;
1940 }
1941
1942 /*
1943 * Lookup the instance, locking only the event queue.
1944 * Can't grab ri_lock here because it might be held
1945 * by a long-running method.
1946 */
1947 rip = inst_lookup_queue(e->rpe_inst);
1948 if (rip == NULL) {
1949 log_error(LOG_INFO, "Restarter: "
1950 "Ignoring %s command for unknown service "
1951 "%s.\n", event_names[e->rpe_type],
1952 e->rpe_inst);
1953 goto nolookup;
1954 }
1955
1956 /* Keep ADMIN events from filling up the queue. */
1957 if (is_admin_event(e->rpe_type) &&
1958 uu_list_numnodes(rip->ri_queue) >
1959 RINST_QUEUE_THRESHOLD) {
1960 MUTEX_UNLOCK(&rip->ri_queue_lock);
1961 log_instance(rip, B_TRUE, "Instance event "
1962 "queue overflow. Dropping administrative "
1963 "request.");
1964 log_framework(LOG_DEBUG, "%s: Instance event "
1965 "queue overflow. Dropping administrative "
1966 "request.\n", rip->ri_i.i_fmri);
1967 goto nolookup;
1968 }
1969
1970 /* Now add the event to the instance queue. */
1971 restarter_queue_event(rip, e);
1972
1973 if (rip->ri_queue_thread == 0) {
1974 /*
1975 * Start a thread if one isn't already
1976 * running.
1977 */
1978 fmri = safe_strdup(e->rpe_inst);
1979 rip->ri_queue_thread = startd_thread_create(
1980 restarter_process_events, (void *)fmri);
1981 } else {
1982 /*
1983 * Signal the existing thread that there's
1984 * a new event.
1985 */
1986 (void) pthread_cond_broadcast(
1987 &rip->ri_queue_cv);
1988 }
1989
1990 MUTEX_UNLOCK(&rip->ri_queue_lock);
1991 nolookup:
1992 restarter_event_release(e);
1993
1994 MUTEX_LOCK(&ru->restarter_update_lock);
1995 }
1996 }
1997
1998 /*
1999 * Unreachable for now -- there's currently no graceful cleanup
2000 * called on exit().
2001 */
2002 (void) scf_handle_unbind(h);
2003 scf_handle_destroy(h);
2004 return (NULL);
2005 }
2006
2007 static restarter_inst_t *
2008 contract_to_inst(ctid_t ctid)
2009 {
2010 restarter_inst_t *inst;
2011 int id;
2012
2013 id = lookup_inst_by_contract(ctid);
2014 if (id == -1)
2015 return (NULL);
2016
2017 inst = inst_lookup_by_id(id);
2018 if (inst != NULL) {
2019 /*
2020 * Since ri_lock isn't held by the contract id lookup, this
2021 * instance may have been restarted and now be in a new
2022 * contract, making the old contract no longer valid for this
2023 * instance.
2024 */
2025 if (ctid != inst->ri_i.i_primary_ctid) {
2026 MUTEX_UNLOCK(&inst->ri_lock);
2027 inst = NULL;
2028 }
2029 }
2030 return (inst);
2031 }
2032
2033 /*
2034 * void contract_action()
2035 * Take action on contract events.
2036 */
2037 static void
2038 contract_action(scf_handle_t *h, restarter_inst_t *inst, ctid_t id,
2039 uint32_t type)
2040 {
2041 const char *fmri = inst->ri_i.i_fmri;
2042
2043 assert(MUTEX_HELD(&inst->ri_lock));
2044
2045 /*
2046 * If startd has stopped this contract, there is no need to
2047 * stop it again.
2048 */
2049 if (inst->ri_i.i_primary_ctid > 0 &&
2050 inst->ri_i.i_primary_ctid_stopped)
2051 return;
2052
2053 if ((type & (CT_PR_EV_EMPTY | CT_PR_EV_CORE | CT_PR_EV_SIGNAL
2054 | CT_PR_EV_HWERR)) == 0) {
2055 /*
2056 * There shouldn't be other events, since that's not how we set
2057 * the terms. Thus, just log an error and drive on.
2058 */
2059 log_framework(LOG_NOTICE,
2060 "%s: contract %ld received unexpected critical event "
2061 "(%d)\n", fmri, id, type);
2062 return;
2063 }
2064
2065 assert(instance_in_transition(inst) == 0);
2066
2067 if (instance_is_wait_style(inst)) {
2068 /*
2069 * We ignore all events; if they impact the
2070 * process we're monitoring, then the
2071 * wait_thread will stop the instance.
2072 */
2073 log_framework(LOG_DEBUG,
2074 "%s: ignoring contract event on wait-style service\n",
2075 fmri);
2076 } else {
2077 /*
2078 * A CT_PR_EV_EMPTY event is an RSTOP_EXIT request.
2079 */
2080 switch (type) {
2081 case CT_PR_EV_EMPTY:
2082 (void) stop_instance(h, inst, RSTOP_EXIT);
2083 break;
2084 case CT_PR_EV_CORE:
2085 (void) stop_instance(h, inst, RSTOP_CORE);
2086 break;
2087 case CT_PR_EV_SIGNAL:
2088 (void) stop_instance(h, inst, RSTOP_SIGNAL);
2089 break;
2090 case CT_PR_EV_HWERR:
2091 (void) stop_instance(h, inst, RSTOP_HWERR);
2092 break;
2093 }
2094 }
2095 }
2096
2097 /*
2098 * void *restarter_contract_event_thread(void *)
2099 * Listens to the process contract bundle for critical events, taking action
2100 * on events from contracts we know we are responsible for.
2101 */
2102 /*ARGSUSED*/
2103 static void *
2104 restarter_contracts_event_thread(void *unused)
2105 {
2106 int fd, err;
2107 scf_handle_t *local_handle;
2108
2109 /*
2110 * Await graph load completion. That is, stop here, until we've scanned
2111 * the repository for contract - instance associations.
2112 */
2113 MUTEX_LOCK(&st->st_load_lock);
2114 while (!(st->st_load_complete && st->st_load_instances == 0))
2115 (void) pthread_cond_wait(&st->st_load_cv, &st->st_load_lock);
2116 MUTEX_UNLOCK(&st->st_load_lock);
2117
2118 /*
2119 * This is a new thread, and thus, gets its own handle
2120 * to the repository.
2121 */
2122 if ((local_handle = libscf_handle_create_bound(SCF_VERSION)) == NULL)
2123 uu_die("Unable to bind a new repository handle: %s\n",
2124 scf_strerror(scf_error()));
2125
2126 fd = open64(CTFS_ROOT "/process/pbundle", O_RDONLY);
2127 if (fd == -1)
2128 uu_die("process bundle open failed");
2129
2130 /*
2131 * Make sure we get all events (including those generated by configd
2132 * before this thread was started).
2133 */
2134 err = ct_event_reset(fd);
2135 assert(err == 0);
2136
2137 for (;;) {
2138 int efd, sfd;
2139 ct_evthdl_t ev;
2140 uint32_t type;
2141 ctevid_t evid;
2142 ct_stathdl_t status;
2143 ctid_t ctid;
2144 restarter_inst_t *inst;
2145 uint64_t cookie;
2146
2147 if (err = ct_event_read_critical(fd, &ev)) {
2148 log_error(LOG_WARNING,
2149 "Error reading next contract event: %s",
2150 strerror(err));
2151 continue;
2152 }
2153
2154 evid = ct_event_get_evid(ev);
2155 ctid = ct_event_get_ctid(ev);
2156 type = ct_event_get_type(ev);
2157
2158 /* Fetch cookie. */
2159 if ((sfd = contract_open(ctid, "process", "status", O_RDONLY))
2160 < 0) {
2161 ct_event_free(ev);
2162 continue;
2163 }
2164
2165 if (err = ct_status_read(sfd, CTD_COMMON, &status)) {
2166 log_framework(LOG_WARNING, "Could not get status for "
2167 "contract %ld: %s\n", ctid, strerror(err));
2168
2169 startd_close(sfd);
2170 ct_event_free(ev);
2171 continue;
2172 }
2173
2174 cookie = ct_status_get_cookie(status);
2175
2176 log_framework(LOG_DEBUG, "Received event %d for ctid %ld "
2177 "cookie %lld\n", type, ctid, cookie);
2178
2179 ct_status_free(status);
2180
2181 startd_close(sfd);
2182
2183 /*
2184 * svc.configd(1M) restart handling performed by the
2185 * fork_configd_thread. We don't acknowledge, as that thread
2186 * will do so.
2187 */
2188 if (cookie == CONFIGD_COOKIE) {
2189 ct_event_free(ev);
2190 continue;
2191 }
2192
2193 inst = NULL;
2194 if (storing_contract != 0 &&
2195 (inst = contract_to_inst(ctid)) == NULL) {
2196 /*
2197 * This can happen for two reasons:
2198 * - method_run() has not yet stored the
2199 * the contract into the internal hash table.
2200 * - we receive an EMPTY event for an abandoned
2201 * contract.
2202 * If there is any contract in the process of
2203 * being stored into the hash table then re-read
2204 * the event later.
2205 */
2206 log_framework(LOG_DEBUG,
2207 "Reset event %d for unknown "
2208 "contract id %ld\n", type, ctid);
2209
2210 /* don't go too fast */
2211 (void) poll(NULL, 0, 100);
2212
2213 (void) ct_event_reset(fd);
2214 ct_event_free(ev);
2215 continue;
2216 }
2217
2218 /*
2219 * Do not call contract_to_inst() again if first
2220 * call succeeded.
2221 */
2222 if (inst == NULL)
2223 inst = contract_to_inst(ctid);
2224 if (inst == NULL) {
2225 /*
2226 * This can happen if we receive an EMPTY
2227 * event for an abandoned contract.
2228 */
2229 log_framework(LOG_DEBUG,
2230 "Received event %d for unknown contract id "
2231 "%ld\n", type, ctid);
2232 } else {
2233 log_framework(LOG_DEBUG,
2234 "Received event %d for contract id "
2235 "%ld (%s)\n", type, ctid,
2236 inst->ri_i.i_fmri);
2237
2238 contract_action(local_handle, inst, ctid, type);
2239
2240 MUTEX_UNLOCK(&inst->ri_lock);
2241 }
2242
2243 efd = contract_open(ct_event_get_ctid(ev), "process", "ctl",
2244 O_WRONLY);
2245 if (efd != -1) {
2246 (void) ct_ctl_ack(efd, evid);
2247 startd_close(efd);
2248 }
2249
2250 ct_event_free(ev);
2251
2252 }
2253
2254 /*NOTREACHED*/
2255 return (NULL);
2256 }
2257
2258 /*
2259 * Timeout queue, processed by restarter_timeouts_event_thread().
2260 */
2261 timeout_queue_t *timeouts;
2262 static uu_list_pool_t *timeout_pool;
2263
2264 typedef struct timeout_update {
2265 pthread_mutex_t tu_lock;
2266 pthread_cond_t tu_cv;
2267 int tu_wakeup;
2268 } timeout_update_t;
2269
2270 timeout_update_t *tu;
2271
2272 static const char *timeout_ovr_svcs[] = {
2273 "svc:/system/manifest-import:default",
2274 "svc:/network/initial:default",
2275 "svc:/network/service:default",
2276 "svc:/system/rmtmpfiles:default",
2277 "svc:/network/loopback:default",
2278 "svc:/network/physical:default",
2279 "svc:/system/device/local:default",
2280 "svc:/system/metainit:default",
2281 "svc:/system/filesystem/usr:default",
2282 "svc:/system/filesystem/minimal:default",
2283 "svc:/system/filesystem/local:default",
2284 NULL
2285 };
2286
2287 int
2288 is_timeout_ovr(restarter_inst_t *inst)
2289 {
2290 int i;
2291
2292 for (i = 0; timeout_ovr_svcs[i] != NULL; ++i) {
2293 if (strcmp(inst->ri_i.i_fmri, timeout_ovr_svcs[i]) == 0) {
2294 log_instance(inst, B_TRUE, "Timeout override by "
2295 "svc.startd. Using infinite timeout.");
2296 return (1);
2297 }
2298 }
2299
2300 return (0);
2301 }
2302
2303 /*ARGSUSED*/
2304 static int
2305 timeout_compare(const void *lc_arg, const void *rc_arg, void *private)
2306 {
2307 hrtime_t t1 = ((const timeout_entry_t *)lc_arg)->te_timeout;
2308 hrtime_t t2 = ((const timeout_entry_t *)rc_arg)->te_timeout;
2309
2310 if (t1 > t2)
2311 return (1);
2312 else if (t1 < t2)
2313 return (-1);
2314 return (0);
2315 }
2316
2317 void
2318 timeout_init()
2319 {
2320 timeouts = startd_zalloc(sizeof (timeout_queue_t));
2321
2322 (void) pthread_mutex_init(&timeouts->tq_lock, &mutex_attrs);
2323
2324 timeout_pool = startd_list_pool_create("timeouts",
2325 sizeof (timeout_entry_t), offsetof(timeout_entry_t, te_link),
2326 timeout_compare, UU_LIST_POOL_DEBUG);
2327 assert(timeout_pool != NULL);
2328
2329 timeouts->tq_list = startd_list_create(timeout_pool,
2330 timeouts, UU_LIST_SORTED);
2331 assert(timeouts->tq_list != NULL);
2332
2333 tu = startd_zalloc(sizeof (timeout_update_t));
2334 (void) pthread_cond_init(&tu->tu_cv, NULL);
2335 (void) pthread_mutex_init(&tu->tu_lock, &mutex_attrs);
2336 }
2337
2338 void
2339 timeout_insert(restarter_inst_t *inst, ctid_t cid, uint64_t timeout_sec)
2340 {
2341 hrtime_t now, timeout;
2342 timeout_entry_t *entry;
2343 uu_list_index_t idx;
2344
2345 assert(MUTEX_HELD(&inst->ri_lock));
2346
2347 now = gethrtime();
2348
2349 /*
2350 * If we overflow LLONG_MAX, we're never timing out anyways, so
2351 * just return.
2352 */
2353 if (timeout_sec >= (LLONG_MAX - now) / 1000000000LL) {
2354 log_instance(inst, B_TRUE, "timeout_seconds too large, "
2355 "treating as infinite.");
2356 return;
2357 }
2358
2359 /* hrtime is in nanoseconds. Convert timeout_sec. */
2360 timeout = now + (timeout_sec * 1000000000LL);
2361
2362 entry = startd_alloc(sizeof (timeout_entry_t));
2363 entry->te_timeout = timeout;
2364 entry->te_ctid = cid;
2365 entry->te_fmri = safe_strdup(inst->ri_i.i_fmri);
2366 entry->te_logstem = safe_strdup(inst->ri_logstem);
2367 entry->te_fired = 0;
2368 /* Insert the calculated timeout time onto the queue. */
2369 MUTEX_LOCK(&timeouts->tq_lock);
2370 (void) uu_list_find(timeouts->tq_list, entry, NULL, &idx);
2371 uu_list_node_init(entry, &entry->te_link, timeout_pool);
2372 uu_list_insert(timeouts->tq_list, entry, idx);
2373 MUTEX_UNLOCK(&timeouts->tq_lock);
2374
2375 assert(inst->ri_timeout == NULL);
2376 inst->ri_timeout = entry;
2377
2378 MUTEX_LOCK(&tu->tu_lock);
2379 tu->tu_wakeup = 1;
2380 (void) pthread_cond_broadcast(&tu->tu_cv);
2381 MUTEX_UNLOCK(&tu->tu_lock);
2382 }
2383
2384
2385 void
2386 timeout_remove(restarter_inst_t *inst, ctid_t cid)
2387 {
2388 assert(MUTEX_HELD(&inst->ri_lock));
2389
2390 if (inst->ri_timeout == NULL)
2391 return;
2392
2393 assert(inst->ri_timeout->te_ctid == cid);
2394
2395 MUTEX_LOCK(&timeouts->tq_lock);
2396 uu_list_remove(timeouts->tq_list, inst->ri_timeout);
2397 MUTEX_UNLOCK(&timeouts->tq_lock);
2398
2399 free(inst->ri_timeout->te_fmri);
2400 free(inst->ri_timeout->te_logstem);
2401 startd_free(inst->ri_timeout, sizeof (timeout_entry_t));
2402 inst->ri_timeout = NULL;
2403 }
2404
2405 static int
2406 timeout_now()
2407 {
2408 timeout_entry_t *e;
2409 hrtime_t now;
2410 int ret;
2411
2412 now = gethrtime();
2413
2414 /*
2415 * Walk through the (sorted) timeouts list. While the timeout
2416 * at the head of the list is <= the current time, kill the
2417 * method.
2418 */
2419 MUTEX_LOCK(&timeouts->tq_lock);
2420
2421 for (e = uu_list_first(timeouts->tq_list);
2422 e != NULL && e->te_timeout <= now;
2423 e = uu_list_next(timeouts->tq_list, e)) {
2424 log_framework(LOG_WARNING, "%s: Method or service exit timed "
2425 "out. Killing contract %ld.\n", e->te_fmri, e->te_ctid);
2426 log_instance_fmri(e->te_fmri, e->te_logstem, B_TRUE,
2427 "Method or service exit timed out. Killing contract %ld.",
2428 e->te_ctid);
2429 e->te_fired = 1;
2430 (void) contract_kill(e->te_ctid, SIGKILL, e->te_fmri);
2431 }
2432
2433 if (uu_list_numnodes(timeouts->tq_list) > 0)
2434 ret = 0;
2435 else
2436 ret = -1;
2437
2438 MUTEX_UNLOCK(&timeouts->tq_lock);
2439
2440 return (ret);
2441 }
2442
2443 /*
2444 * void *restarter_timeouts_event_thread(void *)
2445 * Responsible for monitoring the method timeouts. This thread must
2446 * be started before any methods are called.
2447 */
2448 /*ARGSUSED*/
2449 static void *
2450 restarter_timeouts_event_thread(void *unused)
2451 {
2452 /*
2453 * Timeouts are entered on a priority queue, which is processed by
2454 * this thread. As timeouts are specified in seconds, we'll do
2455 * the necessary processing every second, as long as the queue
2456 * is not empty.
2457 */
2458
2459 /*CONSTCOND*/
2460 while (1) {
2461 /*
2462 * As long as the timeout list isn't empty, process it
2463 * every second.
2464 */
2465 if (timeout_now() == 0) {
2466 (void) sleep(1);
2467 continue;
2468 }
2469
2470 /* The list is empty, wait until we have more timeouts. */
2471 MUTEX_LOCK(&tu->tu_lock);
2472
2473 while (tu->tu_wakeup == 0)
2474 (void) pthread_cond_wait(&tu->tu_cv, &tu->tu_lock);
2475
2476 tu->tu_wakeup = 0;
2477 MUTEX_UNLOCK(&tu->tu_lock);
2478 }
2479
2480 return (NULL);
2481 }
2482
2483 void
2484 restarter_start()
2485 {
2486 (void) startd_thread_create(restarter_timeouts_event_thread, NULL);
2487 (void) startd_thread_create(restarter_event_thread, NULL);
2488 (void) startd_thread_create(restarter_contracts_event_thread, NULL);
2489 (void) startd_thread_create(wait_thread, NULL);
2490 }
2491
2492
2493 void
2494 restarter_init()
2495 {
2496 restarter_instance_pool = startd_list_pool_create("restarter_instances",
2497 sizeof (restarter_inst_t), offsetof(restarter_inst_t,
2498 ri_link), restarter_instance_compare, UU_LIST_POOL_DEBUG);
2499 (void) memset(&instance_list, 0, sizeof (instance_list));
2500
2501 (void) pthread_mutex_init(&instance_list.ril_lock, &mutex_attrs);
2502 instance_list.ril_instance_list = startd_list_create(
2503 restarter_instance_pool, &instance_list, UU_LIST_SORTED);
2504
2505 restarter_queue_pool = startd_list_pool_create(
2506 "restarter_instance_queue", sizeof (restarter_instance_qentry_t),
2507 offsetof(restarter_instance_qentry_t, riq_link), NULL,
2508 UU_LIST_POOL_DEBUG);
2509
2510 contract_list_pool = startd_list_pool_create(
2511 "contract_list", sizeof (contract_entry_t),
2512 offsetof(contract_entry_t, ce_link), NULL,
2513 UU_LIST_POOL_DEBUG);
2514 contract_hash_init();
2515
2516 log_framework(LOG_DEBUG, "Initialized restarter\n");
2517 }