4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2018 Joyent, Inc.
28 * restarter.c - service manipulation
30 * This component manages services whose restarter is svc.startd, the standard
31 * restarter. It translates restarter protocol events from the graph engine
32 * into actions on processes, as a delegated restarter would do.
34 * The master restarter manages a number of always-running threads:
35 * - restarter event thread: events from the graph engine
36 * - timeout thread: thread to fire queued timeouts
37 * - contract thread: thread to handle contract events
38 * - wait thread: thread to handle wait-based services
40 * The other threads are created as-needed:
41 * - per-instance method threads
42 * - per-instance event processing threads
44 * The interaction of all threads must result in the following conditions
45 * being satisfied (on a per-instance basis):
46 * - restarter events must be processed in order
47 * - method execution must be serialized
48 * - instance delete must be held until outstanding methods are complete
49 * - contract events shouldn't be processed while a method is running
50 * - timeouts should fire even when a method is running
52 * Service instances are represented by restarter_inst_t's and are kept in the
56 * The current state of a service instance is kept in
57 * restarter_inst_t->ri_i.i_state. If transition to a new state could take
58 * some time, then before we effect the transition we set
59 * restarter_inst_t->ri_i.i_next_state to the target state, and afterwards we
60 * rotate i_next_state to i_state and set i_next_state to
61 * RESTARTER_STATE_NONE. So usually i_next_state is _NONE when ri_lock is not
62 * held. The exception is when we launch methods, which are done with
63 * a separate thread. To keep any other threads from grabbing ri_lock before
64 * method_thread() does, we set ri_method_thread to the thread id of the
65 * method thread, and when it is nonzero any thread with a different thread id
66 * waits on ri_method_cv.
68 * Method execution is serialized by blocking on ri_method_cv in
69 * inst_lookup_by_id() and waiting for a 0 value of ri_method_thread. This
70 * also prevents the instance structure from being deleted until all
71 * outstanding operations such as method_thread() have finished.
75 * dgraph_lock [can be held when taking:]
77 * dictionary->dict_lock
80 * ru->restarter_update_lock
81 * restarter_queue->rpeq_lock
82 * instance_list.ril_lock
84 * st->st_configd_live_lock
86 * instance_list.ril_lock
87 * graph_queue->gpeq_lock
89 * st->st_configd_live_lock
90 * dictionary->dict_lock
92 * graph_queue->gpeq_lock
101 * single_user_thread_lock
107 * logbuf_mutex nests inside pretty much everything.
110 #include <sys/contract/process.h>
111 #include <sys/ctfs.h>
112 #include <sys/stat.h>
113 #include <sys/time.h>
114 #include <sys/types.h>
116 #include <sys/wait.h>
120 #include <libcontract.h>
121 #include <libcontract_priv.h>
123 #include <librestart.h>
124 #include <librestart_priv.h>
125 #include <libuutil.h>
136 #include "protocol.h"
138 static uu_list_pool_t
*restarter_instance_pool
;
139 static restarter_instance_list_t instance_list
;
141 static uu_list_pool_t
*restarter_queue_pool
;
143 #define WT_SVC_ERR_THROTTLE 1 /* 1 sec delay for erroring wait svc */
146 * Function used to reset the restart times for an instance, when
147 * an administrative task comes along and essentially makes the times
148 * in this array ineffective.
151 reset_start_times(restarter_inst_t
*inst
)
153 inst
->ri_start_index
= 0;
154 bzero(inst
->ri_start_time
, sizeof (inst
->ri_start_time
));
159 restarter_instance_compare(const void *lc_arg
, const void *rc_arg
,
162 int lc_id
= ((const restarter_inst_t
*)lc_arg
)->ri_id
;
163 int rc_id
= *(int *)rc_arg
;
172 static restarter_inst_t
*
173 inst_lookup_by_name(const char *name
)
177 id
= dict_lookup_byname(name
);
181 return (inst_lookup_by_id(id
));
185 inst_lookup_by_id(int id
)
187 restarter_inst_t
*inst
;
189 MUTEX_LOCK(&instance_list
.ril_lock
);
190 inst
= uu_list_find(instance_list
.ril_instance_list
, &id
, NULL
, NULL
);
192 MUTEX_LOCK(&inst
->ri_lock
);
193 MUTEX_UNLOCK(&instance_list
.ril_lock
);
196 while (inst
->ri_method_thread
!= 0 &&
197 !pthread_equal(inst
->ri_method_thread
, pthread_self())) {
198 ++inst
->ri_method_waiters
;
199 (void) pthread_cond_wait(&inst
->ri_method_cv
,
201 assert(inst
->ri_method_waiters
> 0);
202 --inst
->ri_method_waiters
;
209 static restarter_inst_t
*
210 inst_lookup_queue(const char *name
)
213 restarter_inst_t
*inst
;
215 id
= dict_lookup_byname(name
);
219 MUTEX_LOCK(&instance_list
.ril_lock
);
220 inst
= uu_list_find(instance_list
.ril_instance_list
, &id
, NULL
, NULL
);
222 MUTEX_LOCK(&inst
->ri_queue_lock
);
223 MUTEX_UNLOCK(&instance_list
.ril_lock
);
229 service_style(int flags
)
231 switch (flags
& RINST_STYLE_MASK
) {
232 case RINST_CONTRACT
: return ("contract");
233 case RINST_TRANSIENT
: return ("transient");
234 case RINST_WAIT
: return ("wait");
238 uu_warn("%s:%d: Bad flags 0x%x.\n", __FILE__
, __LINE__
, flags
);
246 * Fails with ECONNABORTED or ECANCELED.
249 check_contract(restarter_inst_t
*inst
, boolean_t primary
,
250 scf_instance_t
*scf_inst
)
255 ctidp
= primary
? &inst
->ri_i
.i_primary_ctid
:
256 &inst
->ri_i
.i_transient_ctid
;
260 fd
= contract_open(*ctidp
, NULL
, "status", O_RDONLY
);
267 r
= restarter_remove_contract(scf_inst
, *ctidp
, primary
?
268 RESTARTER_CONTRACT_PRIMARY
: RESTARTER_CONTRACT_TRANSIENT
);
277 uu_die("Out of memory\n");
281 uu_die("Insufficient privilege.\n");
285 uu_die("Repository backend access denied.\n");
289 log_error(LOG_INFO
, "Could not remove unusable contract id %ld "
290 "for %s from repository.\n", *ctidp
, inst
->ri_i
.i_fmri
);
302 static int stop_instance(scf_handle_t
*, restarter_inst_t
*, stop_cause_t
);
305 * int restarter_insert_inst(scf_handle_t *, char *)
306 * If the inst is already in the restarter list, return its id. If the inst
307 * is not in the restarter list, initialize a restarter_inst_t, initialize its
308 * states, insert it into the list, and return 0.
311 * ENOENT - name is not in the repository
314 restarter_insert_inst(scf_handle_t
*h
, const char *name
)
317 restarter_inst_t
*inst
;
319 scf_service_t
*scf_svc
;
320 scf_instance_t
*scf_inst
;
321 scf_snapshot_t
*snap
= NULL
;
322 scf_propertygroup_t
*pg
;
323 char *svc_name
, *inst_name
;
324 char logfilebuf
[PATH_MAX
];
326 boolean_t do_commit_states
;
327 restarter_instance_state_t state
, next_state
;
328 protocol_states_t
*ps
;
330 restarter_str_t reason
= restarter_str_insert_in_graph
;
332 MUTEX_LOCK(&instance_list
.ril_lock
);
335 * We don't use inst_lookup_by_name() here because we want the lookup
336 * & insert to be atomic.
338 id
= dict_lookup_byname(name
);
340 inst
= uu_list_find(instance_list
.ril_instance_list
, &id
, NULL
,
343 MUTEX_UNLOCK(&instance_list
.ril_lock
);
348 /* Allocate an instance */
349 inst
= startd_zalloc(sizeof (restarter_inst_t
));
350 inst
->ri_utmpx_prefix
= startd_alloc(max_scf_value_size
);
351 inst
->ri_utmpx_prefix
[0] = '\0';
353 inst
->ri_i
.i_fmri
= startd_alloc(strlen(name
) + 1);
354 (void) strcpy((char *)inst
->ri_i
.i_fmri
, name
);
356 inst
->ri_queue
= startd_list_create(restarter_queue_pool
, inst
, 0);
359 * id shouldn't be -1 since we use the same dictionary as graph.c, but
362 inst
->ri_id
= (id
!= -1 ? id
: dict_insert(name
));
364 special_online_hooks_get(name
, &inst
->ri_pre_online_hook
,
365 &inst
->ri_post_online_hook
, &inst
->ri_post_offline_hook
);
367 scf_svc
= safe_scf_service_create(h
);
368 scf_inst
= safe_scf_instance_create(h
);
369 pg
= safe_scf_pg_create(h
);
370 svc_name
= startd_alloc(max_scf_name_size
);
371 inst_name
= startd_alloc(max_scf_name_size
);
375 scf_snapshot_destroy(snap
);
376 if (inst
->ri_logstem
!= NULL
)
377 startd_free(inst
->ri_logstem
, PATH_MAX
);
378 if (inst
->ri_common_name
!= NULL
)
379 free(inst
->ri_common_name
);
380 if (inst
->ri_C_common_name
!= NULL
)
381 free(inst
->ri_C_common_name
);
383 inst
->ri_logstem
= NULL
;
384 inst
->ri_common_name
= NULL
;
385 inst
->ri_C_common_name
= NULL
;
387 if (scf_handle_decode_fmri(h
, name
, NULL
, scf_svc
, scf_inst
, NULL
,
388 NULL
, SCF_DECODE_FMRI_EXACT
) != 0) {
389 switch (scf_error()) {
390 case SCF_ERROR_CONNECTION_BROKEN
:
391 libscf_handle_rebind(h
);
394 case SCF_ERROR_NOT_FOUND
:
398 uu_die("Can't decode FMRI %s: %s\n", name
,
399 scf_strerror(scf_error()));
403 * If there's no running snapshot, then we execute using the editing
404 * snapshot. Pending snapshots will be taken later.
406 snap
= libscf_get_running_snapshot(scf_inst
);
408 if ((scf_service_get_name(scf_svc
, svc_name
, max_scf_name_size
) < 0) ||
409 (scf_instance_get_name(scf_inst
, inst_name
, max_scf_name_size
) <
411 switch (scf_error()) {
412 case SCF_ERROR_NOT_SET
:
415 case SCF_ERROR_CONNECTION_BROKEN
:
416 libscf_handle_rebind(h
);
427 (void) snprintf(logfilebuf
, PATH_MAX
, "%s:%s", svc_name
, inst_name
);
428 for (c
= logfilebuf
; *c
!= '\0'; c
++)
432 inst
->ri_logstem
= startd_alloc(PATH_MAX
);
433 (void) snprintf(inst
->ri_logstem
, PATH_MAX
, "%s%s", logfilebuf
,
437 * If the restarter group is missing, use uninit/none. Otherwise,
438 * we're probably being restarted & don't want to mess up the states
441 state
= RESTARTER_STATE_UNINIT
;
442 next_state
= RESTARTER_STATE_NONE
;
444 r
= scf_instance_get_pg(scf_inst
, SCF_PG_RESTARTER
, pg
);
446 switch (scf_error()) {
447 case SCF_ERROR_CONNECTION_BROKEN
:
448 libscf_handle_rebind(h
);
451 case SCF_ERROR_NOT_SET
:
454 case SCF_ERROR_NOT_FOUND
:
456 * This shouldn't happen since the graph engine should
457 * have initialized the state to uninitialized/none if
458 * there was no restarter pg. In case somebody
459 * deleted it, though....
461 do_commit_states
= B_TRUE
;
469 r
= libscf_read_states(pg
, &state
, &next_state
);
471 do_commit_states
= B_TRUE
;
473 if (next_state
!= RESTARTER_STATE_NONE
) {
475 * Force next_state to _NONE since we
476 * don't look for method processes.
478 next_state
= RESTARTER_STATE_NONE
;
479 do_commit_states
= B_TRUE
;
482 * The reason for transition will depend on
485 if (st
->st_initial
== 0)
486 reason
= restarter_str_startd_restart
;
487 else if (state
== RESTARTER_STATE_MAINT
)
488 reason
= restarter_str_bad_repo_state
;
490 * Inform the restarter of our state without
491 * changing the STIME in the repository.
493 ps
= startd_alloc(sizeof (*ps
));
494 inst
->ri_i
.i_state
= ps
->ps_state
= state
;
495 inst
->ri_i
.i_next_state
= ps
->ps_state_next
=
497 ps
->ps_reason
= reason
;
499 graph_protocol_send_event(inst
->ri_i
.i_fmri
,
500 GRAPH_UPDATE_STATE_CHANGE
, ps
);
502 do_commit_states
= B_FALSE
;
507 switch (libscf_get_startd_properties(scf_inst
, snap
, &inst
->ri_flags
,
508 &inst
->ri_utmpx_prefix
)) {
513 libscf_handle_rebind(h
);
521 * This is odd, because the graph engine should have required
522 * the general property group. So we'll just use default
523 * flags in anticipation of the graph engine sending us
524 * REMOVE_INSTANCE when it finds out that the general property
525 * group has been deleted.
527 inst
->ri_flags
= RINST_CONTRACT
;
535 r
= libscf_get_template_values(scf_inst
, snap
,
536 &inst
->ri_common_name
, &inst
->ri_C_common_name
);
539 * Copy our names to smaller buffers to reduce our memory footprint.
541 if (inst
->ri_common_name
!= NULL
) {
542 char *tmp
= safe_strdup(inst
->ri_common_name
);
543 startd_free(inst
->ri_common_name
, max_scf_value_size
);
544 inst
->ri_common_name
= tmp
;
547 if (inst
->ri_C_common_name
!= NULL
) {
548 char *tmp
= safe_strdup(inst
->ri_C_common_name
);
549 startd_free(inst
->ri_C_common_name
, max_scf_value_size
);
550 inst
->ri_C_common_name
= tmp
;
558 libscf_handle_rebind(h
);
573 switch (libscf_read_method_ids(h
, scf_inst
, inst
->ri_i
.i_fmri
,
574 &inst
->ri_i
.i_primary_ctid
, &inst
->ri_i
.i_transient_ctid
,
580 libscf_handle_rebind(h
);
591 if (inst
->ri_i
.i_primary_ctid
>= 1) {
592 contract_hash_store(inst
->ri_i
.i_primary_ctid
, inst
->ri_id
);
594 switch (check_contract(inst
, B_TRUE
, scf_inst
)) {
599 libscf_handle_rebind(h
);
611 if (inst
->ri_i
.i_transient_ctid
>= 1) {
612 switch (check_contract(inst
, B_FALSE
, scf_inst
)) {
617 libscf_handle_rebind(h
);
629 /* No more failures we live through, so add it to the list. */
630 (void) pthread_mutex_init(&inst
->ri_lock
, &mutex_attrs
);
631 (void) pthread_mutex_init(&inst
->ri_queue_lock
, &mutex_attrs
);
632 MUTEX_LOCK(&inst
->ri_lock
);
633 MUTEX_LOCK(&inst
->ri_queue_lock
);
635 (void) pthread_cond_init(&inst
->ri_method_cv
, NULL
);
637 uu_list_node_init(inst
, &inst
->ri_link
, restarter_instance_pool
);
638 uu_list_insert(instance_list
.ril_instance_list
, inst
, idx
);
639 MUTEX_UNLOCK(&instance_list
.ril_lock
);
641 if (start_pid
!= -1 &&
642 (inst
->ri_flags
& RINST_STYLE_MASK
) == RINST_WAIT
) {
644 ret
= wait_register(start_pid
, inst
->ri_i
.i_fmri
, 0, 1);
647 * Implication: if we can't reregister the
648 * instance, we will start another one. Two
649 * instances may or may not result in a resource
652 log_error(LOG_WARNING
,
653 "%s: couldn't reregister %ld for wait\n",
654 inst
->ri_i
.i_fmri
, start_pid
);
655 } else if (ret
== 1) {
657 * Leading PID has exited.
659 (void) stop_instance(h
, inst
, RSTOP_EXIT
);
666 if (do_commit_states
)
667 (void) restarter_instance_update_states(h
, inst
, state
,
668 next_state
, RERR_NONE
, reason
);
670 log_framework(LOG_DEBUG
, "%s is a %s-style service\n", name
,
671 service_style(inst
->ri_flags
));
673 MUTEX_UNLOCK(&inst
->ri_queue_lock
);
674 MUTEX_UNLOCK(&inst
->ri_lock
);
676 startd_free(svc_name
, max_scf_name_size
);
677 startd_free(inst_name
, max_scf_name_size
);
678 scf_snapshot_destroy(snap
);
679 scf_instance_destroy(scf_inst
);
680 scf_service_destroy(scf_svc
);
682 log_framework(LOG_DEBUG
, "%s: inserted instance into restarter list\n",
688 MUTEX_UNLOCK(&instance_list
.ril_lock
);
689 startd_free(inst_name
, max_scf_name_size
);
690 startd_free(svc_name
, max_scf_name_size
);
692 scf_snapshot_destroy(snap
);
694 scf_instance_destroy(scf_inst
);
695 scf_service_destroy(scf_svc
);
696 startd_free((void *)inst
->ri_i
.i_fmri
, strlen(inst
->ri_i
.i_fmri
) + 1);
697 uu_list_destroy(inst
->ri_queue
);
698 if (inst
->ri_logstem
!= NULL
)
699 startd_free(inst
->ri_logstem
, PATH_MAX
);
700 if (inst
->ri_common_name
!= NULL
)
701 free(inst
->ri_common_name
);
702 if (inst
->ri_C_common_name
!= NULL
)
703 free(inst
->ri_C_common_name
);
704 startd_free(inst
->ri_utmpx_prefix
, max_scf_value_size
);
705 startd_free(inst
, sizeof (restarter_inst_t
));
710 restarter_delete_inst(restarter_inst_t
*ri
)
713 restarter_inst_t
*rip
;
715 restarter_instance_qentry_t
*e
;
717 assert(MUTEX_HELD(&ri
->ri_lock
));
720 * Must drop the instance lock so we can pick up the instance_list
721 * lock & remove the instance.
724 MUTEX_UNLOCK(&ri
->ri_lock
);
726 MUTEX_LOCK(&instance_list
.ril_lock
);
728 rip
= uu_list_find(instance_list
.ril_instance_list
, &id
, NULL
, NULL
);
730 MUTEX_UNLOCK(&instance_list
.ril_lock
);
736 uu_list_remove(instance_list
.ril_instance_list
, ri
);
738 log_framework(LOG_DEBUG
, "%s: deleted instance from restarter list\n",
741 MUTEX_UNLOCK(&instance_list
.ril_lock
);
744 * We can lock the instance without holding the instance_list lock
745 * since we removed the instance from the list.
747 MUTEX_LOCK(&ri
->ri_lock
);
748 MUTEX_LOCK(&ri
->ri_queue_lock
);
750 if (ri
->ri_i
.i_primary_ctid
>= 1)
751 contract_hash_remove(ri
->ri_i
.i_primary_ctid
);
753 while (ri
->ri_method_thread
!= 0 || ri
->ri_method_waiters
> 0)
754 (void) pthread_cond_wait(&ri
->ri_method_cv
, &ri
->ri_lock
);
756 while ((e
= uu_list_teardown(ri
->ri_queue
, &cookie
)) != NULL
)
757 startd_free(e
, sizeof (*e
));
758 uu_list_destroy(ri
->ri_queue
);
760 startd_free((void *)ri
->ri_i
.i_fmri
, strlen(ri
->ri_i
.i_fmri
) + 1);
761 startd_free(ri
->ri_logstem
, PATH_MAX
);
762 if (ri
->ri_common_name
!= NULL
)
763 free(ri
->ri_common_name
);
764 if (ri
->ri_C_common_name
!= NULL
)
765 free(ri
->ri_C_common_name
);
766 startd_free(ri
->ri_utmpx_prefix
, max_scf_value_size
);
767 (void) pthread_mutex_destroy(&ri
->ri_lock
);
768 (void) pthread_mutex_destroy(&ri
->ri_queue_lock
);
769 startd_free(ri
, sizeof (restarter_inst_t
));
773 * instance_is_wait_style()
775 * Returns 1 if the given instance is a "wait-style" service instance.
778 instance_is_wait_style(restarter_inst_t
*inst
)
780 assert(MUTEX_HELD(&inst
->ri_lock
));
781 return ((inst
->ri_flags
& RINST_STYLE_MASK
) == RINST_WAIT
);
785 * instance_is_transient_style()
787 * Returns 1 if the given instance is a transient service instance.
790 instance_is_transient_style(restarter_inst_t
*inst
)
792 assert(MUTEX_HELD(&inst
->ri_lock
));
793 return ((inst
->ri_flags
& RINST_STYLE_MASK
) == RINST_TRANSIENT
);
797 * instance_in_transition()
798 * Returns 1 if instance is in transition, 0 if not
801 instance_in_transition(restarter_inst_t
*inst
)
803 assert(MUTEX_HELD(&inst
->ri_lock
));
804 if (inst
->ri_i
.i_next_state
== RESTARTER_STATE_NONE
)
810 * returns 1 if instance is already started, 0 if not
813 instance_started(restarter_inst_t
*inst
)
817 assert(MUTEX_HELD(&inst
->ri_lock
));
819 if (inst
->ri_i
.i_state
== RESTARTER_STATE_ONLINE
||
820 inst
->ri_i
.i_state
== RESTARTER_STATE_DEGRADED
)
831 * ECONNRESET - success, but h was rebound
834 restarter_instance_update_states(scf_handle_t
*h
, restarter_inst_t
*ri
,
835 restarter_instance_state_t new_state
,
836 restarter_instance_state_t new_state_next
, restarter_error_t err
,
837 restarter_str_t reason
)
839 protocol_states_t
*states
;
841 uint_t retry_count
= 0, msecs
= ALLOC_DELAY
;
842 boolean_t rebound
= B_FALSE
;
843 int prev_state_online
;
846 assert(MUTEX_HELD(&ri
->ri_lock
));
848 prev_state_online
= instance_started(ri
);
851 e
= _restarter_commit_states(h
, &ri
->ri_i
, new_state
, new_state_next
,
852 restarter_get_str_short(reason
));
859 if (retry_count
< ALLOC_RETRY
) {
860 (void) poll(NULL
, 0, msecs
);
861 msecs
*= ALLOC_DELAY_MULT
;
865 /* Like startd_alloc(). */
866 uu_die("Insufficient memory.\n");
870 libscf_handle_rebind(h
);
877 log_error(LOG_NOTICE
, "Could not commit state change for %s "
878 "to repository: %s.\n", ri
->ri_i
.i_fmri
, strerror(e
));
882 ri
->ri_i
.i_state
= new_state
;
883 ri
->ri_i
.i_next_state
= new_state_next
;
888 bad_error("_restarter_commit_states", e
);
891 states
= startd_alloc(sizeof (protocol_states_t
));
892 states
->ps_state
= new_state
;
893 states
->ps_state_next
= new_state_next
;
894 states
->ps_err
= err
;
895 states
->ps_reason
= reason
;
896 graph_protocol_send_event(ri
->ri_i
.i_fmri
, GRAPH_UPDATE_STATE_CHANGE
,
899 state_online
= instance_started(ri
);
901 if (prev_state_online
&& !state_online
)
902 ri
->ri_post_offline_hook();
903 else if (!prev_state_online
&& state_online
)
904 ri
->ri_post_online_hook();
906 return (rebound
? ECONNRESET
: 0);
910 restarter_mark_pending_snapshot(const char *fmri
, uint_t flag
)
912 restarter_inst_t
*inst
;
914 assert(flag
== RINST_RETAKE_RUNNING
|| flag
== RINST_RETAKE_START
);
916 inst
= inst_lookup_by_name(fmri
);
920 inst
->ri_flags
|= flag
;
922 MUTEX_UNLOCK(&inst
->ri_lock
);
926 restarter_take_pending_snapshots(scf_handle_t
*h
)
928 restarter_inst_t
*inst
;
931 MUTEX_LOCK(&instance_list
.ril_lock
);
933 for (inst
= uu_list_first(instance_list
.ril_instance_list
);
935 inst
= uu_list_next(instance_list
.ril_instance_list
, inst
)) {
937 scf_instance_t
*sinst
= NULL
;
939 MUTEX_LOCK(&inst
->ri_lock
);
942 * This is where we'd check inst->ri_method_thread and if it
943 * were nonzero we'd wait in anticipation of another thread
944 * executing a method for inst. Doing so with the instance_list
945 * locked, though, leads to deadlock. Since taking a snapshot
946 * during that window won't hurt anything, we'll just continue.
949 fmri
= inst
->ri_i
.i_fmri
;
951 if (inst
->ri_flags
& RINST_RETAKE_RUNNING
) {
952 scf_snapshot_t
*rsnap
;
954 (void) libscf_fmri_get_instance(h
, fmri
, &sinst
);
956 rsnap
= libscf_get_or_make_running_snapshot(sinst
,
959 scf_instance_destroy(sinst
);
962 inst
->ri_flags
&= ~RINST_RETAKE_RUNNING
;
964 scf_snapshot_destroy(rsnap
);
967 if (inst
->ri_flags
& RINST_RETAKE_START
) {
968 switch (r
= libscf_snapshots_poststart(h
, fmri
,
972 inst
->ri_flags
&= ~RINST_RETAKE_START
;
980 bad_error("libscf_snapshots_poststart", r
);
984 MUTEX_UNLOCK(&inst
->ri_lock
);
987 MUTEX_UNLOCK(&instance_list
.ril_lock
);
992 restarter_post_fsminimal_thread(void *unused
)
997 (void) pthread_setname_np(pthread_self(), "restarter_post_fsmin");
999 h
= libscf_handle_create_bound_loop();
1002 r
= libscf_create_self(h
);
1006 assert(r
== ECONNABORTED
);
1007 libscf_handle_rebind(h
);
1010 restarter_take_pending_snapshots(h
);
1012 (void) scf_handle_unbind(h
);
1013 scf_handle_destroy(h
);
1019 * int stop_instance()
1021 * Stop the instance identified by the instance given as the second argument,
1022 * for the cause stated.
1026 * -1 - inst is in transition
1029 stop_instance(scf_handle_t
*local_handle
, restarter_inst_t
*inst
,
1035 restarter_error_t re
;
1036 restarter_str_t reason
;
1037 restarter_instance_state_t new_state
;
1039 assert(MUTEX_HELD(&inst
->ri_lock
));
1040 assert(inst
->ri_method_thread
== 0);
1045 reason
= restarter_str_ct_ev_exit
;
1046 cp
= "all processes in service exited";
1050 reason
= restarter_str_method_failed
;
1051 cp
= "service exited with a configuration error";
1053 case RSTOP_ERR_EXIT
:
1055 reason
= restarter_str_ct_ev_exit
;
1056 cp
= "service exited with an error";
1060 reason
= restarter_str_ct_ev_core
;
1061 cp
= "process dumped core";
1065 reason
= restarter_str_ct_ev_signal
;
1066 cp
= "process received fatal signal from outside the service";
1070 reason
= restarter_str_ct_ev_hwerr
;
1071 cp
= "process killed due to uncorrectable hardware error";
1073 case RSTOP_DEPENDENCY
:
1075 reason
= restarter_str_dependency_activity
;
1076 cp
= "dependency activity requires stop";
1080 reason
= restarter_str_disable_request
;
1081 cp
= "service disabled";
1085 reason
= restarter_str_restart_request
;
1086 cp
= "service restarting";
1090 (void) fprintf(stderr
, "Unknown cause %d at %s:%d.\n",
1091 cause
, __FILE__
, __LINE__
);
1096 /* Services in the disabled and maintenance state are ignored */
1097 if (inst
->ri_i
.i_state
== RESTARTER_STATE_MAINT
||
1098 inst
->ri_i
.i_state
== RESTARTER_STATE_DISABLED
) {
1099 log_framework(LOG_DEBUG
,
1100 "%s: stop_instance -> is maint/disabled\n",
1105 /* Already stopped instances are left alone */
1106 if (instance_started(inst
) == 0) {
1107 log_framework(LOG_DEBUG
, "Restarter: %s is already stopped.\n",
1112 if (instance_in_transition(inst
)) {
1113 /* requeue event by returning -1 */
1114 log_framework(LOG_DEBUG
,
1115 "Restarter: Not stopping %s, in transition.\n",
1120 log_instance(inst
, B_TRUE
, "Stopping because %s.", cp
);
1122 log_framework(re
== RERR_FAULT
? LOG_INFO
: LOG_DEBUG
,
1123 "%s: Instance stopping because %s.\n", inst
->ri_i
.i_fmri
, cp
);
1125 if (instance_is_wait_style(inst
) &&
1126 (cause
== RSTOP_EXIT
||
1127 cause
== RSTOP_ERR_CFG
||
1128 cause
== RSTOP_ERR_EXIT
)) {
1130 * No need to stop instance, as child has exited; remove
1131 * contract and move the instance to the offline state.
1133 switch (err
= restarter_instance_update_states(local_handle
,
1134 inst
, inst
->ri_i
.i_state
, RESTARTER_STATE_OFFLINE
, re
,
1141 bad_error("restarter_instance_update_states", err
);
1144 if (cause
== RSTOP_ERR_EXIT
) {
1146 * The RSTOP_ERR_EXIT cause is set via the
1147 * wait_thread -> wait_remove code path when we have
1148 * a "wait" style svc that exited with an error. If
1149 * the svc is failing too quickly, we throttle it so
1150 * that we don't restart it more than once/second.
1151 * Since we know we're running in the wait thread its
1152 * ok to throttle it right here.
1154 (void) update_fault_count(inst
, FAULT_COUNT_INCR
);
1155 if (method_rate_critical(inst
)) {
1156 log_instance(inst
, B_TRUE
, "Failing too "
1157 "quickly, throttling.");
1158 (void) sleep(WT_SVC_ERR_THROTTLE
);
1161 (void) update_fault_count(inst
, FAULT_COUNT_RESET
);
1162 reset_start_times(inst
);
1165 if (inst
->ri_i
.i_primary_ctid
!= 0) {
1167 safe_scf_instance_create(local_handle
);
1168 inst
->ri_mi_deleted
= B_FALSE
;
1170 libscf_reget_instance(inst
);
1171 method_remove_contract(inst
, B_TRUE
, B_TRUE
);
1173 scf_instance_destroy(inst
->ri_m_inst
);
1174 inst
->ri_m_inst
= NULL
;
1177 switch (err
= restarter_instance_update_states(local_handle
,
1178 inst
, inst
->ri_i
.i_next_state
, RESTARTER_STATE_NONE
, re
,
1185 bad_error("restarter_instance_update_states", err
);
1188 if (cause
!= RSTOP_ERR_CFG
)
1190 } else if (instance_is_wait_style(inst
) && re
== RERR_RESTART
) {
1192 * Stopping a wait service through means other than the pid
1193 * exiting should keep wait_thread() from restarting the
1194 * service, by removing it from the wait list.
1195 * We cannot remove it right now otherwise the process will
1196 * end up <defunct> so mark it to be ignored.
1198 wait_ignore_by_fmri(inst
->ri_i
.i_fmri
);
1202 * There are some configuration errors which we cannot detect until we
1203 * try to run the method. For example, see exec_method() where the
1204 * restarter_set_method_context() call can return SMF_EXIT_ERR_CONFIG
1205 * in several cases. If this happens for a "wait-style" svc,
1206 * wait_remove() sets the cause as RSTOP_ERR_CFG so that we can detect
1207 * the configuration error and go into maintenance, even though it is
1208 * a "wait-style" svc.
1210 if (cause
== RSTOP_ERR_CFG
)
1211 new_state
= RESTARTER_STATE_MAINT
;
1213 new_state
= inst
->ri_i
.i_enabled
?
1214 RESTARTER_STATE_OFFLINE
: RESTARTER_STATE_DISABLED
;
1216 switch (err
= restarter_instance_update_states(local_handle
, inst
,
1217 inst
->ri_i
.i_state
, new_state
, RERR_NONE
, reason
)) {
1223 bad_error("restarter_instance_update_states", err
);
1226 info
= startd_zalloc(sizeof (fork_info_t
));
1228 info
->sf_id
= inst
->ri_id
;
1229 info
->sf_method_type
= METHOD_STOP
;
1230 info
->sf_event_type
= re
;
1231 info
->sf_reason
= reason
;
1232 inst
->ri_method_thread
= startd_thread_create(method_thread
, info
);
1239 * ENOENT - fmri is not in instance_list
1241 * ECONNRESET - success, though handle was rebound
1242 * -1 - instance is in transition
1245 stop_instance_fmri(scf_handle_t
*h
, const char *fmri
, uint_t flags
)
1247 restarter_inst_t
*rip
;
1250 rip
= inst_lookup_by_name(fmri
);
1254 r
= stop_instance(h
, rip
, flags
);
1256 MUTEX_UNLOCK(&rip
->ri_lock
);
1262 unmaintain_instance(scf_handle_t
*h
, restarter_inst_t
*rip
,
1263 unmaint_cause_t cause
)
1266 scf_instance_t
*inst
;
1268 uint_t tries
= 0, msecs
= ALLOC_DELAY
;
1270 restarter_str_t reason
;
1272 assert(MUTEX_HELD(&rip
->ri_lock
));
1274 if (rip
->ri_i
.i_state
!= RESTARTER_STATE_MAINT
) {
1275 log_error(LOG_DEBUG
, "Restarter: "
1276 "Ignoring maintenance off command because %s is not in the "
1277 "maintenance state.\n", rip
->ri_i
.i_fmri
);
1282 case RUNMAINT_CLEAR
:
1283 cp
= "clear requested";
1284 reason
= restarter_str_clear_request
;
1286 case RUNMAINT_DISABLE
:
1287 cp
= "disable requested";
1288 reason
= restarter_str_disable_request
;
1292 (void) fprintf(stderr
, "Uncaught case for %d at %s:%d.\n",
1293 cause
, __FILE__
, __LINE__
);
1298 log_instance(rip
, B_TRUE
, "Leaving maintenance because %s.",
1300 log_framework(LOG_DEBUG
, "%s: Instance leaving maintenance because "
1301 "%s.\n", rip
->ri_i
.i_fmri
, cp
);
1303 (void) restarter_instance_update_states(h
, rip
, RESTARTER_STATE_UNINIT
,
1304 RESTARTER_STATE_NONE
, RERR_RESTART
, reason
);
1307 * If we did ADMIN_MAINT_ON_IMMEDIATE, then there might still be
1308 * a primary contract.
1310 if (rip
->ri_i
.i_primary_ctid
== 0)
1313 ctid
= rip
->ri_i
.i_primary_ctid
;
1314 contract_abandon(ctid
);
1315 rip
->ri_i
.i_primary_ctid
= 0;
1318 switch (r
= libscf_fmri_get_instance(h
, rip
->ri_i
.i_fmri
, &inst
)) {
1323 libscf_handle_rebind(h
);
1327 /* Must have been deleted. */
1333 bad_error("libscf_handle_rebind", r
);
1337 r
= restarter_remove_contract(inst
, ctid
, RESTARTER_CONTRACT_PRIMARY
);
1344 if (tries
< ALLOC_RETRY
) {
1345 (void) poll(NULL
, 0, msecs
);
1346 msecs
*= ALLOC_DELAY_MULT
;
1350 uu_die("Insufficient memory.\n");
1354 scf_instance_destroy(inst
);
1355 libscf_handle_rebind(h
);
1365 "Could not remove contract id %lu for %s (%s).\n", ctid
,
1366 rip
->ri_i
.i_fmri
, strerror(r
));
1372 bad_error("restarter_remove_contract", r
);
1375 scf_instance_destroy(inst
);
1380 * Set inst->ri_i.i_enabled. Expects 'e' to be _ENABLE, _DISABLE, or
1381 * _ADMIN_DISABLE. If the event is _ENABLE and inst is uninitialized or
1382 * disabled, move it to offline. If the event is _DISABLE or
1383 * _ADMIN_DISABLE, make sure inst will move to disabled.
1387 * ECONNRESET - h was rebound
1390 enable_inst(scf_handle_t
*h
, restarter_inst_t
*inst
,
1391 restarter_instance_qentry_t
*riq
)
1393 restarter_instance_state_t state
;
1394 restarter_event_type_t e
= riq
->riq_type
;
1395 restarter_str_t reason
= restarter_str_per_configuration
;
1398 assert(MUTEX_HELD(&inst
->ri_lock
));
1399 assert(e
== RESTARTER_EVENT_TYPE_ADMIN_DISABLE
||
1400 e
== RESTARTER_EVENT_TYPE_DISABLE
||
1401 e
== RESTARTER_EVENT_TYPE_ENABLE
);
1402 assert(instance_in_transition(inst
) == 0);
1404 state
= inst
->ri_i
.i_state
;
1406 if (e
== RESTARTER_EVENT_TYPE_ENABLE
) {
1407 inst
->ri_i
.i_enabled
= 1;
1409 if (state
== RESTARTER_STATE_UNINIT
||
1410 state
== RESTARTER_STATE_DISABLED
) {
1412 * B_FALSE: Don't log an error if the log_instance()
1413 * fails because it will fail on the miniroot before
1414 * install-discovery runs.
1416 log_instance(inst
, B_FALSE
, "Enabled.");
1417 log_framework(LOG_DEBUG
, "%s: Instance enabled.\n",
1421 * If we are coming from DISABLED, it was obviously an
1422 * enable request. If we are coming from UNINIT, it may
1423 * have been a sevice in MAINT that was cleared.
1425 if (riq
->riq_reason
== restarter_str_clear_request
)
1426 reason
= restarter_str_clear_request
;
1427 else if (state
== RESTARTER_STATE_DISABLED
)
1428 reason
= restarter_str_enable_request
;
1429 (void) restarter_instance_update_states(h
, inst
,
1430 RESTARTER_STATE_OFFLINE
, RESTARTER_STATE_NONE
,
1433 log_framework(LOG_DEBUG
, "Restarter: "
1434 "Not changing state of %s for enable command.\n",
1438 inst
->ri_i
.i_enabled
= 0;
1441 case RESTARTER_STATE_ONLINE
:
1442 case RESTARTER_STATE_DEGRADED
:
1443 r
= stop_instance(h
, inst
, RSTOP_DISABLE
);
1444 return (r
== ECONNRESET
? 0 : r
);
1446 case RESTARTER_STATE_OFFLINE
:
1447 case RESTARTER_STATE_UNINIT
:
1448 if (inst
->ri_i
.i_primary_ctid
!= 0) {
1449 inst
->ri_m_inst
= safe_scf_instance_create(h
);
1450 inst
->ri_mi_deleted
= B_FALSE
;
1452 libscf_reget_instance(inst
);
1453 method_remove_contract(inst
, B_TRUE
, B_TRUE
);
1455 scf_instance_destroy(inst
->ri_m_inst
);
1457 /* B_FALSE: See log_instance(..., "Enabled."); above */
1458 log_instance(inst
, B_FALSE
, "Disabled.");
1459 log_framework(LOG_DEBUG
, "%s: Instance disabled.\n",
1463 * If we are coming from OFFLINE, it was obviously a
1464 * disable request. But if we are coming from
1465 * UNINIT, it may have been a disable request for a
1468 if (riq
->riq_reason
== restarter_str_disable_request
||
1469 state
== RESTARTER_STATE_OFFLINE
)
1470 reason
= restarter_str_disable_request
;
1471 (void) restarter_instance_update_states(h
, inst
,
1472 RESTARTER_STATE_DISABLED
, RESTARTER_STATE_NONE
,
1473 RERR_RESTART
, reason
);
1476 case RESTARTER_STATE_DISABLED
:
1479 case RESTARTER_STATE_MAINT
:
1481 * We only want to pull the instance out of maintenance
1482 * if the disable is on adminstrative request. The
1483 * graph engine sends _DISABLE events whenever a
1484 * service isn't in the disabled state, and we don't
1485 * want to pull the service out of maintenance if,
1486 * for example, it is there due to a dependency cycle.
1488 if (e
== RESTARTER_EVENT_TYPE_ADMIN_DISABLE
)
1489 unmaintain_instance(h
, inst
, RUNMAINT_DISABLE
);
1494 (void) fprintf(stderr
, "Restarter instance %s has "
1495 "unknown state %d.\n", inst
->ri_i
.i_fmri
, state
);
1505 start_instance(scf_handle_t
*local_handle
, restarter_inst_t
*inst
,
1509 restarter_str_t new_reason
;
1511 assert(MUTEX_HELD(&inst
->ri_lock
));
1512 assert(instance_in_transition(inst
) == 0);
1513 assert(inst
->ri_method_thread
== 0);
1515 log_framework(LOG_DEBUG
, "%s: trying to start instance\n",
1519 * We want to keep the original reason for restarts and clear actions
1522 case restarter_str_restart_request
:
1523 case restarter_str_clear_request
:
1524 new_reason
= reason
;
1527 new_reason
= restarter_str_dependencies_satisfied
;
1530 /* Services in the disabled and maintenance state are ignored */
1531 if (inst
->ri_i
.i_state
== RESTARTER_STATE_MAINT
||
1532 inst
->ri_i
.i_state
== RESTARTER_STATE_DISABLED
||
1533 inst
->ri_i
.i_enabled
== 0) {
1534 log_framework(LOG_DEBUG
,
1535 "%s: start_instance -> is maint/disabled\n",
1540 /* Already started instances are left alone */
1541 if (instance_started(inst
) == 1) {
1542 log_framework(LOG_DEBUG
,
1543 "%s: start_instance -> is already started\n",
1548 log_framework(LOG_DEBUG
, "%s: starting instance.\n", inst
->ri_i
.i_fmri
);
1550 (void) restarter_instance_update_states(local_handle
, inst
,
1551 inst
->ri_i
.i_state
, RESTARTER_STATE_ONLINE
, RERR_NONE
, new_reason
);
1553 info
= startd_zalloc(sizeof (fork_info_t
));
1555 info
->sf_id
= inst
->ri_id
;
1556 info
->sf_method_type
= METHOD_START
;
1557 info
->sf_event_type
= RERR_NONE
;
1558 info
->sf_reason
= new_reason
;
1559 inst
->ri_method_thread
= startd_thread_create(method_thread
, info
);
1563 event_from_tty(scf_handle_t
*h
, restarter_inst_t
*rip
)
1565 scf_instance_t
*inst
;
1568 if (libscf_fmri_get_instance(h
, rip
->ri_i
.i_fmri
, &inst
))
1571 ret
= restarter_inst_ractions_from_tty(inst
);
1573 scf_instance_destroy(inst
);
1578 restart_dump(scf_handle_t
*h
, restarter_inst_t
*rip
)
1580 scf_instance_t
*inst
;
1581 boolean_t ret
= B_FALSE
;
1583 if (libscf_fmri_get_instance(h
, rip
->ri_i
.i_fmri
, &inst
))
1586 if (restarter_inst_dump(inst
) == 1)
1589 scf_instance_destroy(inst
);
1594 maintain_instance(scf_handle_t
*h
, restarter_inst_t
*rip
, int immediate
,
1595 restarter_str_t reason
)
1598 scf_instance_t
*scf_inst
= NULL
;
1600 assert(MUTEX_HELD(&rip
->ri_lock
));
1601 assert(reason
!= restarter_str_none
);
1602 assert(rip
->ri_method_thread
== 0);
1604 log_instance(rip
, B_TRUE
, "Stopping for maintenance due to %s.",
1605 restarter_get_str_short(reason
));
1606 log_framework(LOG_DEBUG
, "%s: stopping for maintenance due to %s.\n",
1607 rip
->ri_i
.i_fmri
, restarter_get_str_short(reason
));
1609 /* Services in the maintenance state are ignored */
1610 if (rip
->ri_i
.i_state
== RESTARTER_STATE_MAINT
) {
1611 log_framework(LOG_DEBUG
,
1612 "%s: maintain_instance -> is already in maintenance\n",
1618 * If reason state is restarter_str_service_request and
1619 * restarter_actions/auxiliary_fmri property is set with a valid fmri,
1620 * copy the fmri to restarter/auxiliary_fmri so svcs -x can use.
1622 if (reason
== restarter_str_service_request
&&
1623 libscf_fmri_get_instance(h
, rip
->ri_i
.i_fmri
, &scf_inst
) == 0) {
1624 if (restarter_inst_validate_ractions_aux_fmri(scf_inst
) == 0) {
1625 if (restarter_inst_set_aux_fmri(scf_inst
))
1626 log_framework(LOG_DEBUG
, "%s: "
1627 "restarter_inst_set_aux_fmri failed: ",
1630 log_framework(LOG_DEBUG
, "%s: "
1631 "restarter_inst_validate_ractions_aux_fmri "
1632 "failed: ", rip
->ri_i
.i_fmri
);
1634 if (restarter_inst_reset_aux_fmri(scf_inst
))
1635 log_framework(LOG_DEBUG
, "%s: "
1636 "restarter_inst_reset_aux_fmri failed: ",
1639 scf_instance_destroy(scf_inst
);
1642 if (immediate
|| !instance_started(rip
)) {
1643 if (rip
->ri_i
.i_primary_ctid
!= 0) {
1644 rip
->ri_m_inst
= safe_scf_instance_create(h
);
1645 rip
->ri_mi_deleted
= B_FALSE
;
1647 libscf_reget_instance(rip
);
1648 method_remove_contract(rip
, B_TRUE
, B_TRUE
);
1650 scf_instance_destroy(rip
->ri_m_inst
);
1653 (void) restarter_instance_update_states(h
, rip
,
1654 RESTARTER_STATE_MAINT
, RESTARTER_STATE_NONE
, RERR_RESTART
,
1659 (void) restarter_instance_update_states(h
, rip
, rip
->ri_i
.i_state
,
1660 RESTARTER_STATE_MAINT
, RERR_NONE
, reason
);
1662 log_transition(rip
, MAINT_REQUESTED
);
1664 info
= startd_zalloc(sizeof (*info
));
1665 info
->sf_id
= rip
->ri_id
;
1666 info
->sf_method_type
= METHOD_STOP
;
1667 info
->sf_event_type
= RERR_RESTART
;
1668 info
->sf_reason
= reason
;
1669 rip
->ri_method_thread
= startd_thread_create(method_thread
, info
);
1673 refresh_instance(scf_handle_t
*h
, restarter_inst_t
*rip
)
1675 scf_instance_t
*inst
;
1676 scf_snapshot_t
*snap
;
1680 assert(MUTEX_HELD(&rip
->ri_lock
));
1682 log_instance(rip
, B_TRUE
, "Rereading configuration.");
1683 log_framework(LOG_DEBUG
, "%s: rereading configuration.\n",
1687 r
= libscf_fmri_get_instance(h
, rip
->ri_i
.i_fmri
, &inst
);
1693 libscf_handle_rebind(h
);
1697 /* Must have been deleted. */
1703 bad_error("libscf_fmri_get_instance", r
);
1706 snap
= libscf_get_running_snapshot(inst
);
1708 r
= libscf_get_startd_properties(inst
, snap
, &rip
->ri_flags
,
1709 &rip
->ri_utmpx_prefix
);
1712 log_framework(LOG_DEBUG
, "%s is a %s-style service\n",
1713 rip
->ri_i
.i_fmri
, service_style(rip
->ri_flags
));
1717 scf_instance_destroy(inst
);
1718 scf_snapshot_destroy(snap
);
1719 libscf_handle_rebind(h
);
1724 /* Succeed in anticipation of REMOVE_INSTANCE. */
1728 bad_error("libscf_get_startd_properties", r
);
1731 if (instance_started(rip
)) {
1732 /* Refresh does not change the state. */
1733 (void) restarter_instance_update_states(h
, rip
,
1734 rip
->ri_i
.i_state
, rip
->ri_i
.i_state
, RERR_NONE
,
1735 restarter_str_refresh
);
1737 info
= startd_zalloc(sizeof (*info
));
1738 info
->sf_id
= rip
->ri_id
;
1739 info
->sf_method_type
= METHOD_REFRESH
;
1740 info
->sf_event_type
= RERR_REFRESH
;
1741 info
->sf_reason
= NULL
;
1743 assert(rip
->ri_method_thread
== 0);
1744 rip
->ri_method_thread
=
1745 startd_thread_create(method_thread
, info
);
1748 scf_snapshot_destroy(snap
);
1749 scf_instance_destroy(inst
);
1752 const char *event_names
[] = { "INVALID", "ADD_INSTANCE", "REMOVE_INSTANCE",
1753 "ENABLE", "DISABLE", "ADMIN_DEGRADED", "ADMIN_REFRESH",
1754 "ADMIN_RESTART", "ADMIN_MAINT_OFF", "ADMIN_MAINT_ON",
1755 "ADMIN_MAINT_ON_IMMEDIATE", "STOP", "START", "DEPENDENCY_CYCLE",
1756 "INVALID_DEPENDENCY", "ADMIN_DISABLE", "STOP_RESET"
1760 * void *restarter_process_events()
1762 * Called in a separate thread to process the events on an instance's
1763 * queue. Empties the queue completely, and tries to keep the thread
1764 * around for a little while after the queue is empty to save on
1768 restarter_process_events(void *arg
)
1771 restarter_instance_qentry_t
*event
;
1772 restarter_inst_t
*rip
;
1773 char *fmri
= (char *)arg
;
1776 (void) pthread_setname_np(pthread_self(), "restarter_process_events");
1778 assert(fmri
!= NULL
);
1780 h
= libscf_handle_create_bound_loop();
1782 /* grab the queue lock */
1783 rip
= inst_lookup_queue(fmri
);
1789 while ((event
= uu_list_first(rip
->ri_queue
)) != NULL
) {
1790 restarter_inst_t
*inst
;
1792 /* drop the queue lock */
1793 MUTEX_UNLOCK(&rip
->ri_queue_lock
);
1796 * Grab the inst lock -- this waits until any outstanding
1797 * method finishes running.
1799 inst
= inst_lookup_by_name(fmri
);
1801 /* Getting deleted in the middle isn't an error. */
1805 assert(instance_in_transition(inst
) == 0);
1807 /* process the event */
1808 switch (event
->riq_type
) {
1809 case RESTARTER_EVENT_TYPE_ENABLE
:
1810 case RESTARTER_EVENT_TYPE_DISABLE
:
1811 (void) enable_inst(h
, inst
, event
);
1814 case RESTARTER_EVENT_TYPE_ADMIN_DISABLE
:
1815 if (enable_inst(h
, inst
, event
) == 0)
1816 reset_start_times(inst
);
1819 case RESTARTER_EVENT_TYPE_REMOVE_INSTANCE
:
1820 restarter_delete_inst(inst
);
1824 case RESTARTER_EVENT_TYPE_STOP_RESET
:
1825 reset_start_times(inst
);
1827 case RESTARTER_EVENT_TYPE_STOP
:
1828 (void) stop_instance(h
, inst
, RSTOP_DEPENDENCY
);
1831 case RESTARTER_EVENT_TYPE_START
:
1832 start_instance(h
, inst
, event
->riq_reason
);
1835 case RESTARTER_EVENT_TYPE_DEPENDENCY_CYCLE
:
1836 maintain_instance(h
, inst
, 0,
1837 restarter_str_dependency_cycle
);
1840 case RESTARTER_EVENT_TYPE_INVALID_DEPENDENCY
:
1841 maintain_instance(h
, inst
, 0,
1842 restarter_str_invalid_dependency
);
1845 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON
:
1846 if (event_from_tty(h
, inst
) == 0)
1847 maintain_instance(h
, inst
, 0,
1848 restarter_str_service_request
);
1850 maintain_instance(h
, inst
, 0,
1851 restarter_str_administrative_request
);
1854 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE
:
1855 if (event_from_tty(h
, inst
) == 0)
1856 maintain_instance(h
, inst
, 1,
1857 restarter_str_service_request
);
1859 maintain_instance(h
, inst
, 1,
1860 restarter_str_administrative_request
);
1863 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF
:
1864 unmaintain_instance(h
, inst
, RUNMAINT_CLEAR
);
1865 reset_start_times(inst
);
1868 case RESTARTER_EVENT_TYPE_ADMIN_REFRESH
:
1869 refresh_instance(h
, inst
);
1872 case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED
:
1873 log_framework(LOG_WARNING
, "Restarter: "
1874 "%s command (for %s) unimplemented.\n",
1875 event_names
[event
->riq_type
], inst
->ri_i
.i_fmri
);
1878 case RESTARTER_EVENT_TYPE_ADMIN_RESTART
:
1879 if (!instance_started(inst
)) {
1880 log_framework(LOG_DEBUG
, "Restarter: "
1881 "Not restarting %s; not running.\n",
1885 * Stop the instance. If it can be restarted,
1886 * the graph engine will send a new event.
1888 if (restart_dump(h
, inst
)) {
1889 (void) contract_kill(
1890 inst
->ri_i
.i_primary_ctid
, SIGABRT
,
1892 } else if (stop_instance(h
, inst
,
1893 RSTOP_RESTART
) == 0) {
1894 reset_start_times(inst
);
1899 case RESTARTER_EVENT_TYPE_ADD_INSTANCE
:
1902 uu_warn("%s:%d: Bad restarter event %d. "
1903 "Aborting.\n", __FILE__
, __LINE__
, event
->riq_type
);
1908 assert(inst
!= NULL
);
1909 MUTEX_UNLOCK(&inst
->ri_lock
);
1912 /* grab the queue lock */
1913 rip
= inst_lookup_queue(fmri
);
1917 /* delete the event */
1918 uu_list_remove(rip
->ri_queue
, event
);
1919 startd_free(event
, sizeof (restarter_instance_qentry_t
));
1922 assert(rip
!= NULL
);
1925 * Try to preserve the thread for a little while for future use.
1929 (void) pthread_cond_reltimedwait_np(&rip
->ri_queue_cv
,
1930 &rip
->ri_queue_lock
, &to
);
1932 if (uu_list_first(rip
->ri_queue
) != NULL
)
1935 rip
->ri_queue_thread
= 0;
1936 MUTEX_UNLOCK(&rip
->ri_queue_lock
);
1939 (void) scf_handle_unbind(h
);
1940 scf_handle_destroy(h
);
1946 is_admin_event(restarter_event_type_t t
)
1949 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON
:
1950 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_ON_IMMEDIATE
:
1951 case RESTARTER_EVENT_TYPE_ADMIN_MAINT_OFF
:
1952 case RESTARTER_EVENT_TYPE_ADMIN_REFRESH
:
1953 case RESTARTER_EVENT_TYPE_ADMIN_DEGRADED
:
1954 case RESTARTER_EVENT_TYPE_ADMIN_RESTART
:
1962 restarter_queue_event(restarter_inst_t
*ri
, restarter_protocol_event_t
*e
)
1964 restarter_instance_qentry_t
*qe
;
1967 assert(MUTEX_HELD(&ri
->ri_queue_lock
));
1968 assert(!MUTEX_HELD(&ri
->ri_lock
));
1970 qe
= startd_zalloc(sizeof (restarter_instance_qentry_t
));
1971 qe
->riq_type
= e
->rpe_type
;
1972 qe
->riq_reason
= e
->rpe_reason
;
1974 uu_list_node_init(qe
, &qe
->riq_link
, restarter_queue_pool
);
1975 r
= uu_list_insert_before(ri
->ri_queue
, NULL
, qe
);
1980 * void *restarter_event_thread()
1982 * Handle incoming graph events by placing them on a per-instance
1983 * queue. We can't lock the main part of the instance structure, so
1984 * just modify the seprarately locked event queue portion.
1988 restarter_event_thread(void *unused
)
1992 (void) pthread_setname_np(pthread_self(), "restarter_event");
1995 * This is a new thread, and thus, gets its own handle
1996 * to the repository.
1998 h
= libscf_handle_create_bound_loop();
2000 MUTEX_LOCK(&ru
->restarter_update_lock
);
2004 restarter_protocol_event_t
*e
;
2006 while (ru
->restarter_update_wakeup
== 0)
2007 (void) pthread_cond_wait(&ru
->restarter_update_cv
,
2008 &ru
->restarter_update_lock
);
2010 ru
->restarter_update_wakeup
= 0;
2012 while ((e
= restarter_event_dequeue()) != NULL
) {
2013 restarter_inst_t
*rip
;
2016 MUTEX_UNLOCK(&ru
->restarter_update_lock
);
2019 * ADD_INSTANCE is special: there's likely no
2020 * instance structure yet, so we need to handle the
2021 * addition synchronously.
2023 switch (e
->rpe_type
) {
2024 case RESTARTER_EVENT_TYPE_ADD_INSTANCE
:
2025 if (restarter_insert_inst(h
, e
->rpe_inst
) != 0)
2026 log_error(LOG_INFO
, "Restarter: "
2027 "Could not add %s.\n", e
->rpe_inst
);
2029 MUTEX_LOCK(&st
->st_load_lock
);
2030 if (--st
->st_load_instances
== 0)
2031 (void) pthread_cond_broadcast(
2033 MUTEX_UNLOCK(&st
->st_load_lock
);
2039 * Lookup the instance, locking only the event queue.
2040 * Can't grab ri_lock here because it might be held
2041 * by a long-running method.
2043 rip
= inst_lookup_queue(e
->rpe_inst
);
2045 log_error(LOG_INFO
, "Restarter: "
2046 "Ignoring %s command for unknown service "
2047 "%s.\n", event_names
[e
->rpe_type
],
2052 /* Keep ADMIN events from filling up the queue. */
2053 if (is_admin_event(e
->rpe_type
) &&
2054 uu_list_numnodes(rip
->ri_queue
) >
2055 RINST_QUEUE_THRESHOLD
) {
2056 MUTEX_UNLOCK(&rip
->ri_queue_lock
);
2057 log_instance(rip
, B_TRUE
, "Instance event "
2058 "queue overflow. Dropping administrative "
2060 log_framework(LOG_DEBUG
, "%s: Instance event "
2061 "queue overflow. Dropping administrative "
2062 "request.\n", rip
->ri_i
.i_fmri
);
2066 /* Now add the event to the instance queue. */
2067 restarter_queue_event(rip
, e
);
2069 if (rip
->ri_queue_thread
== 0) {
2071 * Start a thread if one isn't already
2074 fmri
= safe_strdup(e
->rpe_inst
);
2075 rip
->ri_queue_thread
= startd_thread_create(
2076 restarter_process_events
, (void *)fmri
);
2079 * Signal the existing thread that there's
2082 (void) pthread_cond_broadcast(
2086 MUTEX_UNLOCK(&rip
->ri_queue_lock
);
2088 restarter_event_release(e
);
2090 MUTEX_LOCK(&ru
->restarter_update_lock
);
2095 * Unreachable for now -- there's currently no graceful cleanup
2098 (void) scf_handle_unbind(h
);
2099 scf_handle_destroy(h
);
2103 static restarter_inst_t
*
2104 contract_to_inst(ctid_t ctid
)
2106 restarter_inst_t
*inst
;
2109 id
= lookup_inst_by_contract(ctid
);
2113 inst
= inst_lookup_by_id(id
);
2116 * Since ri_lock isn't held by the contract id lookup, this
2117 * instance may have been restarted and now be in a new
2118 * contract, making the old contract no longer valid for this
2121 if (ctid
!= inst
->ri_i
.i_primary_ctid
) {
2122 MUTEX_UNLOCK(&inst
->ri_lock
);
2130 * void contract_action()
2131 * Take action on contract events.
2134 contract_action(scf_handle_t
*h
, restarter_inst_t
*inst
, ctid_t id
,
2137 const char *fmri
= inst
->ri_i
.i_fmri
;
2139 assert(MUTEX_HELD(&inst
->ri_lock
));
2142 * If startd has stopped this contract, there is no need to
2145 if (inst
->ri_i
.i_primary_ctid
> 0 &&
2146 inst
->ri_i
.i_primary_ctid_stopped
)
2149 if ((type
& (CT_PR_EV_EMPTY
| CT_PR_EV_CORE
| CT_PR_EV_SIGNAL
2150 | CT_PR_EV_HWERR
)) == 0) {
2152 * There shouldn't be other events, since that's not how we set
2153 * the terms. Thus, just log an error and drive on.
2155 log_framework(LOG_NOTICE
,
2156 "%s: contract %ld received unexpected critical event "
2157 "(%d)\n", fmri
, id
, type
);
2161 assert(instance_in_transition(inst
) == 0);
2163 if (instance_is_wait_style(inst
)) {
2165 * We ignore all events; if they impact the
2166 * process we're monitoring, then the
2167 * wait_thread will stop the instance.
2169 log_framework(LOG_DEBUG
,
2170 "%s: ignoring contract event on wait-style service\n",
2174 * A CT_PR_EV_EMPTY event is an RSTOP_EXIT request.
2177 case CT_PR_EV_EMPTY
:
2178 (void) stop_instance(h
, inst
, RSTOP_EXIT
);
2181 (void) stop_instance(h
, inst
, RSTOP_CORE
);
2183 case CT_PR_EV_SIGNAL
:
2184 (void) stop_instance(h
, inst
, RSTOP_SIGNAL
);
2186 case CT_PR_EV_HWERR
:
2187 (void) stop_instance(h
, inst
, RSTOP_HWERR
);
2194 * void *restarter_contract_event_thread(void *)
2195 * Listens to the process contract bundle for critical events, taking action
2196 * on events from contracts we know we are responsible for.
2200 restarter_contracts_event_thread(void *unused
)
2203 scf_handle_t
*local_handle
;
2205 (void) pthread_setname_np(pthread_self(), "restarter_contracts_event");
2208 * Await graph load completion. That is, stop here, until we've scanned
2209 * the repository for contract - instance associations.
2211 MUTEX_LOCK(&st
->st_load_lock
);
2212 while (!(st
->st_load_complete
&& st
->st_load_instances
== 0))
2213 (void) pthread_cond_wait(&st
->st_load_cv
, &st
->st_load_lock
);
2214 MUTEX_UNLOCK(&st
->st_load_lock
);
2217 * This is a new thread, and thus, gets its own handle
2218 * to the repository.
2220 if ((local_handle
= libscf_handle_create_bound(SCF_VERSION
)) == NULL
)
2221 uu_die("Unable to bind a new repository handle: %s\n",
2222 scf_strerror(scf_error()));
2224 fd
= open64(CTFS_ROOT
"/process/pbundle", O_RDONLY
);
2226 uu_die("process bundle open failed");
2229 * Make sure we get all events (including those generated by configd
2230 * before this thread was started).
2232 err
= ct_event_reset(fd
);
2240 ct_stathdl_t status
;
2242 restarter_inst_t
*inst
;
2245 if (err
= ct_event_read_critical(fd
, &ev
)) {
2246 log_error(LOG_WARNING
,
2247 "Error reading next contract event: %s",
2252 evid
= ct_event_get_evid(ev
);
2253 ctid
= ct_event_get_ctid(ev
);
2254 type
= ct_event_get_type(ev
);
2257 if ((sfd
= contract_open(ctid
, "process", "status", O_RDONLY
))
2263 if (err
= ct_status_read(sfd
, CTD_COMMON
, &status
)) {
2264 log_framework(LOG_WARNING
, "Could not get status for "
2265 "contract %ld: %s\n", ctid
, strerror(err
));
2272 cookie
= ct_status_get_cookie(status
);
2274 log_framework(LOG_DEBUG
, "Received event %d for ctid %ld "
2275 "cookie %lld\n", type
, ctid
, cookie
);
2277 ct_status_free(status
);
2282 * svc.configd(1M) restart handling performed by the
2283 * fork_configd_thread. We don't acknowledge, as that thread
2286 if (cookie
== CONFIGD_COOKIE
) {
2292 if (storing_contract
!= 0 &&
2293 (inst
= contract_to_inst(ctid
)) == NULL
) {
2295 * This can happen for two reasons:
2296 * - method_run() has not yet stored the
2297 * the contract into the internal hash table.
2298 * - we receive an EMPTY event for an abandoned
2300 * If there is any contract in the process of
2301 * being stored into the hash table then re-read
2304 log_framework(LOG_DEBUG
,
2305 "Reset event %d for unknown "
2306 "contract id %ld\n", type
, ctid
);
2308 /* don't go too fast */
2309 (void) poll(NULL
, 0, 100);
2311 (void) ct_event_reset(fd
);
2317 * Do not call contract_to_inst() again if first
2321 inst
= contract_to_inst(ctid
);
2324 * This can happen if we receive an EMPTY
2325 * event for an abandoned contract.
2327 log_framework(LOG_DEBUG
,
2328 "Received event %d for unknown contract id "
2329 "%ld\n", type
, ctid
);
2331 log_framework(LOG_DEBUG
,
2332 "Received event %d for contract id "
2333 "%ld (%s)\n", type
, ctid
,
2336 contract_action(local_handle
, inst
, ctid
, type
);
2338 MUTEX_UNLOCK(&inst
->ri_lock
);
2341 efd
= contract_open(ct_event_get_ctid(ev
), "process", "ctl",
2344 (void) ct_ctl_ack(efd
, evid
);
2357 * Timeout queue, processed by restarter_timeouts_event_thread().
2359 timeout_queue_t
*timeouts
;
2360 static uu_list_pool_t
*timeout_pool
;
2362 typedef struct timeout_update
{
2363 pthread_mutex_t tu_lock
;
2364 pthread_cond_t tu_cv
;
2368 timeout_update_t
*tu
;
2370 static const char *timeout_ovr_svcs
[] = {
2371 "svc:/system/manifest-import:default",
2372 "svc:/network/initial:default",
2373 "svc:/network/service:default",
2374 "svc:/system/rmtmpfiles:default",
2375 "svc:/network/loopback:default",
2376 "svc:/network/physical:default",
2377 "svc:/system/device/local:default",
2378 "svc:/system/filesystem/usr:default",
2379 "svc:/system/filesystem/minimal:default",
2380 "svc:/system/filesystem/local:default",
2385 is_timeout_ovr(restarter_inst_t
*inst
)
2389 for (i
= 0; timeout_ovr_svcs
[i
] != NULL
; ++i
) {
2390 if (strcmp(inst
->ri_i
.i_fmri
, timeout_ovr_svcs
[i
]) == 0) {
2391 log_instance(inst
, B_TRUE
, "Timeout override by "
2392 "svc.startd. Using infinite timeout.");
2402 timeout_compare(const void *lc_arg
, const void *rc_arg
, void *private)
2404 hrtime_t t1
= ((const timeout_entry_t
*)lc_arg
)->te_timeout
;
2405 hrtime_t t2
= ((const timeout_entry_t
*)rc_arg
)->te_timeout
;
2417 timeouts
= startd_zalloc(sizeof (timeout_queue_t
));
2419 (void) pthread_mutex_init(&timeouts
->tq_lock
, &mutex_attrs
);
2421 timeout_pool
= startd_list_pool_create("timeouts",
2422 sizeof (timeout_entry_t
), offsetof(timeout_entry_t
, te_link
),
2423 timeout_compare
, UU_LIST_POOL_DEBUG
);
2424 assert(timeout_pool
!= NULL
);
2426 timeouts
->tq_list
= startd_list_create(timeout_pool
,
2427 timeouts
, UU_LIST_SORTED
);
2428 assert(timeouts
->tq_list
!= NULL
);
2430 tu
= startd_zalloc(sizeof (timeout_update_t
));
2431 (void) pthread_cond_init(&tu
->tu_cv
, NULL
);
2432 (void) pthread_mutex_init(&tu
->tu_lock
, &mutex_attrs
);
2436 timeout_insert(restarter_inst_t
*inst
, ctid_t cid
, uint64_t timeout_sec
)
2438 hrtime_t now
, timeout
;
2439 timeout_entry_t
*entry
;
2440 uu_list_index_t idx
;
2442 assert(MUTEX_HELD(&inst
->ri_lock
));
2447 * If we overflow LLONG_MAX, we're never timing out anyways, so
2450 if (timeout_sec
>= (LLONG_MAX
- now
) / 1000000000LL) {
2451 log_instance(inst
, B_TRUE
, "timeout_seconds too large, "
2452 "treating as infinite.");
2456 /* hrtime is in nanoseconds. Convert timeout_sec. */
2457 timeout
= now
+ (timeout_sec
* 1000000000LL);
2459 entry
= startd_alloc(sizeof (timeout_entry_t
));
2460 entry
->te_timeout
= timeout
;
2461 entry
->te_ctid
= cid
;
2462 entry
->te_fmri
= safe_strdup(inst
->ri_i
.i_fmri
);
2463 entry
->te_logstem
= safe_strdup(inst
->ri_logstem
);
2464 entry
->te_fired
= 0;
2465 /* Insert the calculated timeout time onto the queue. */
2466 MUTEX_LOCK(&timeouts
->tq_lock
);
2467 (void) uu_list_find(timeouts
->tq_list
, entry
, NULL
, &idx
);
2468 uu_list_node_init(entry
, &entry
->te_link
, timeout_pool
);
2469 uu_list_insert(timeouts
->tq_list
, entry
, idx
);
2470 MUTEX_UNLOCK(&timeouts
->tq_lock
);
2472 assert(inst
->ri_timeout
== NULL
);
2473 inst
->ri_timeout
= entry
;
2475 MUTEX_LOCK(&tu
->tu_lock
);
2477 (void) pthread_cond_broadcast(&tu
->tu_cv
);
2478 MUTEX_UNLOCK(&tu
->tu_lock
);
2483 timeout_remove(restarter_inst_t
*inst
, ctid_t cid
)
2485 assert(MUTEX_HELD(&inst
->ri_lock
));
2487 if (inst
->ri_timeout
== NULL
)
2490 assert(inst
->ri_timeout
->te_ctid
== cid
);
2492 MUTEX_LOCK(&timeouts
->tq_lock
);
2493 uu_list_remove(timeouts
->tq_list
, inst
->ri_timeout
);
2494 MUTEX_UNLOCK(&timeouts
->tq_lock
);
2496 free(inst
->ri_timeout
->te_fmri
);
2497 free(inst
->ri_timeout
->te_logstem
);
2498 startd_free(inst
->ri_timeout
, sizeof (timeout_entry_t
));
2499 inst
->ri_timeout
= NULL
;
2512 * Walk through the (sorted) timeouts list. While the timeout
2513 * at the head of the list is <= the current time, kill the
2516 MUTEX_LOCK(&timeouts
->tq_lock
);
2518 for (e
= uu_list_first(timeouts
->tq_list
);
2519 e
!= NULL
&& e
->te_timeout
<= now
;
2520 e
= uu_list_next(timeouts
->tq_list
, e
)) {
2521 log_framework(LOG_WARNING
, "%s: Method or service exit timed "
2522 "out. Killing contract %ld.\n", e
->te_fmri
, e
->te_ctid
);
2523 log_instance_fmri(e
->te_fmri
, e
->te_logstem
, B_TRUE
,
2524 "Method or service exit timed out. Killing contract %ld.",
2527 (void) contract_kill(e
->te_ctid
, SIGKILL
, e
->te_fmri
);
2530 if (uu_list_numnodes(timeouts
->tq_list
) > 0)
2535 MUTEX_UNLOCK(&timeouts
->tq_lock
);
2541 * void *restarter_timeouts_event_thread(void *)
2542 * Responsible for monitoring the method timeouts. This thread must
2543 * be started before any methods are called.
2547 restarter_timeouts_event_thread(void *unused
)
2550 * Timeouts are entered on a priority queue, which is processed by
2551 * this thread. As timeouts are specified in seconds, we'll do
2552 * the necessary processing every second, as long as the queue
2556 (void) pthread_setname_np(pthread_self(), "restarter_timeouts_event");
2561 * As long as the timeout list isn't empty, process it
2564 if (timeout_now() == 0) {
2569 /* The list is empty, wait until we have more timeouts. */
2570 MUTEX_LOCK(&tu
->tu_lock
);
2572 while (tu
->tu_wakeup
== 0)
2573 (void) pthread_cond_wait(&tu
->tu_cv
, &tu
->tu_lock
);
2576 MUTEX_UNLOCK(&tu
->tu_lock
);
2585 (void) startd_thread_create(restarter_timeouts_event_thread
, NULL
);
2586 (void) startd_thread_create(restarter_event_thread
, NULL
);
2587 (void) startd_thread_create(restarter_contracts_event_thread
, NULL
);
2588 (void) startd_thread_create(wait_thread
, NULL
);
2595 restarter_instance_pool
= startd_list_pool_create("restarter_instances",
2596 sizeof (restarter_inst_t
), offsetof(restarter_inst_t
,
2597 ri_link
), restarter_instance_compare
, UU_LIST_POOL_DEBUG
);
2598 (void) memset(&instance_list
, 0, sizeof (instance_list
));
2600 (void) pthread_mutex_init(&instance_list
.ril_lock
, &mutex_attrs
);
2601 instance_list
.ril_instance_list
= startd_list_create(
2602 restarter_instance_pool
, &instance_list
, UU_LIST_SORTED
);
2604 restarter_queue_pool
= startd_list_pool_create(
2605 "restarter_instance_queue", sizeof (restarter_instance_qentry_t
),
2606 offsetof(restarter_instance_qentry_t
, riq_link
), NULL
,
2607 UU_LIST_POOL_DEBUG
);
2609 contract_list_pool
= startd_list_pool_create(
2610 "contract_list", sizeof (contract_entry_t
),
2611 offsetof(contract_entry_t
, ce_link
), NULL
,
2612 UU_LIST_POOL_DEBUG
);
2613 contract_hash_init();
2615 log_framework(LOG_DEBUG
, "Initialized restarter\n");