3 * Copyright (c) 2002-2006 MontaVista Software, Inc.
4 * Author: Steven Dake (sdake@mvista.com)
6 * Copyright (c) 2006 Ericsson AB.
7 * Author: Hans Feldt, Anders Eriksson, Lars Holm
8 * - Introduced AMF B.02 information model
9 * - Use DN in API and multicast messages
10 * - (Re-)Introduction of event based multicast messages
11 * - Refactoring of code into several AMF files
12 * - Component/SU restart, SU failover
13 * - Constructors/destructors
14 * - Serializers/deserializers
16 * All rights reserved.
19 * This software licensed under BSD license, the text of which follows:
21 * Redistribution and use in source and binary forms, with or without
22 * modification, are permitted provided that the following conditions are met:
24 * - Redistributions of source code must retain the above copyright notice,
25 * this list of conditions and the following disclaimer.
26 * - Redistributions in binary form must reproduce the above copyright notice,
27 * this list of conditions and the following disclaimer in the documentation
28 * and/or other materials provided with the distribution.
29 * - Neither the name of the MontaVista Software, Inc. nor the names of its
30 * contributors may be used to endorse or promote products derived from this
31 * software without specific prior written permission.
33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
34 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
35 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
36 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
37 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
38 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
39 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
40 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
41 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
42 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
43 * THE POSSIBILITY OF SUCH DAMAGE.
45 * AMF Service Group Class Implementation
47 * This file contains functions for handling AMF-service groups(SGs). It can be
48 * viewed as the implementation of the AMF Service Group class (called SG)
49 * as described in SAI-Overview-B.02.01. The SA Forum specification
50 * SAI-AIS-AMF-B.02.01 has been used as specification of the behaviour
51 * and is referred to as 'the spec' below.
53 * The functions in this file are responsible for:
54 * -on request start the service group by instantiating the contained SUs
55 * -on request assign the service instances it protects to the in-service
56 * service units it contains respecting as many as possible of the configured
57 * requirements for the group
58 * -create and delete an SI-assignment object for each relation between
60 * -order each contained SU to create and delete CSI-assignments
61 * -request the Service Instance class (SI) to execute the transfer of the
62 * HA-state set/remove requests to each component involved
63 * -fully control the execution of component failover and SU failover
64 * -on request control the execution of the initial steps of node switchover
66 * -fully handle the auto adjust procedure
68 * Currently only the 'n+m' redundancy model is implemented. It is the
69 * ambition to identify n+m specific variables and functions and add the suffix
70 * '_nplusm' to them so that they can be easily recognized.
72 * When SG is requested to assign workload to all SUs or all SUs hosted on
73 * a specific node, a procedure containing several steps is executed:
74 * <1> An algorithm is executed which assigns SIs to SUs respecting the rules
75 * that has been configured for SG. The algorithm also has to consider
76 * if assignments between som SIs and SUs already exist. The scope of this
77 * algorithm is to create SI-assignments and set up requested HA-state for
78 * each assignment but not to transfer those HA-states to the components.
79 * <2> All SI-assignments with a requested HA state == ACTIVE are transferred
80 * to the components concerned before any STANDBY assignments are
81 * transferred. All components have to acknowledge the setting of the
82 * ACTIVE HA state before the transfer of any STANDBY assignment is
84 * <3> All active assignments can not be transferred at the same time to the
85 * different components because the rules for dependencies between SI and
86 * SI cluster wide and CSI and CSI within one SI, has to be respected.
88 * SG is fully responsible for step <1> but not fully responsible for handling
89 * step <2> and <3>. However, SG uses an attribute called 'dependency level'
90 * when requsted to assign workload. This parameter refers to an integer that
91 * has been calculated initially for each SI. The 'dependency level' indicates
92 * to which extent an SI depends on other SIs such that an SI that depends on
93 * no other SI is on dependecy_level == 1, an SI that depends only on an SI on
94 * dependency_level == 1 is on dependency-level == 2.
95 * An SI that depends on several SIs gets a
96 * dependency_level that is one unit higher than the SI with the highest
97 * dependency_level it depends on. When SG is requested to assign the workload
98 * on a certain dependency level, it requests all SI objects on that level to
99 * activate (all) SI-assignments that during step <1> has been requested to
100 * assume the active HA state.
102 * SG contains the following state machines:
103 * - administrative state machine (ADSM) (NOT IN THIS RELEASE)
104 * - availability control state machine (ACSM)
106 * The availability control state machine contains three states and one of them
107 * is composite. Being a composite state means that it contains substates.
109 * - IDLE (non composite state)
110 * - INSTANTIATING_SERVICE_UNITS
111 * - MANAGING_SG (composite state)
112 * MANAGING_SG is entered at several different events which has in common
113 * the need to set up or change the assignment of SIs to SUs. Only one such
114 * event can be handled at the time. If new events occur while one event is
115 * being handled then the new event is saved and will be handled after the
116 * handling of the first event is ready (return to IDLE state has been done).
117 * MANAGING_SG handles the following events:
118 * - start (requests SG to order SU to instantiate all SUs in SG and waits
119 * for SU to indicate presence state change reports from the SUs and
120 * finally responds 'started' to the requester)
121 * - assign_si (requests SG to assign SIs to SUs according to pre-configured
122 * rules (if not already done) and transfer the HA state of
123 * the SIs on the requested SI dependency level. Then SG waits for
124 * confirmation that the HA state has been succesfully set and
125 * finally responds 'assigned' to the reqeuster)
126 * - auto_adjust (this event indicates that the auto-adjust probation timer has
127 * expired and that SG should evaluate current assignments of
128 * SIs to SUs and if needed remove current assignments and
129 * create new according to what is specified in paragraph
131 * - failover_comp (requests SG to failover a specific component according to
132 * the procedure described in paragraph 3.12.1.3)
133 * - failover_su (requests SG to failover a specific SU according to the
134 * procedure described in paragraph 3.12.1.3 and 3.12.1.4)
135 * - switchover_node (requests SG to execute the recovery actions described
136 * in 3.12.1.3 and respond to the requester when recovery
138 * - failover_node (requests SG to execute the recovery actions described
139 * in 3.12.1.3 and respond to the requester when recovery is
142 * 1. SG Availability Control State Machine
143 * ==========================================
145 * 1.1 State Transition Table
147 * State: Event: Action: New state:
148 * ============================================================================
149 * IDLE start A48,A28 INSTANTIATING_SUs
150 * IDLE assign_si A48,A31 ASSIGNING_ON_REQ
151 * IDLE failover_su A48,[C22]A10,A11 DEACTIVATING_DEP
152 * IDLE failover_su A48,[!C22]A12 TERMINATING_SUSP
153 * IDLE failover_node A48,[!C22]A12 TERMINATING_SUSP
154 * IDLE failover_node A48,[C22]A10,A11 DEACTIVATING_DEP
155 * IDLE failover_node A48,[C100]A34 IDLE
156 * INSTANTIATING_SUs start A48,A28 INSTANTIATING_SUs
157 * INSTANTIATING_SUs su_state_chg [C101]A26,A53 IDLE
158 * INSTANTIATING_SUs su_state_chg [C102]A26,A53 IDLE
159 * INSTANTIATING_SUs assign_si A31 ASSIGNING_ON_REQ
160 * INSTANTIATING_SUs failover_su A52 INSTANTIATING_SUs
161 * INSTANTIATING_SUs failover_node A52 INSTANTIATING_SUs
162 * ASSIGNING_ON_REQ ha_state_assumed [C15]A54 IDLE
163 * ASSIGNING_ON_REQ failover_su A52 ASSIGNING_ON_REQ
164 * ASSIGNING_ON_REQ failover_node A52 ASSIGNING_ON_REQ
165 * DEACTIVATING_DEP si_deactivated [C20] REMOVING_S-BY_ASS
166 * DEACTIVATING_DEP si_deactivated [!C20]A12 TERMINATING_SUSP
167 * TERMINATING_SUSP su_state_chg [C103]A24,A20 ASS_S-BY_TO_SPARE
168 * TERMINATING_SUSP su_state_chg [C104]A24,A50 REMOVING_S-BY_ASS
169 * TERMINATING_SUSP su_state_chg [C105]A16,A17 ACTIVATING_S-BY
170 * TERMINATING_SUSP su_state_chg [C106]A20 ASS_S-BY_TO_SPARE
171 * TERMINATING_SUSP su_state_chg [C108]A23 REPAIRING_SU
172 * TERMINATING_SUSP su_state_chg [C109] IDLE
173 * TERMINATING_SUSP failover_su A52 TERMINATING_SUSP
174 * TERMINATING_SUSP failover_node A52 TERMINATING_SUSP
175 * REMOVING_S-BY_ASS assignment_removed A51 REMOVING_S-BY_ASS
176 * REMOVING_S-BY_ASS assignment_removed [C27]&[C24] ACTIVATING_S-BY
177 * REMOVING_S-BY_ASS assignment_removed [C110]A20 ASS_S-BY_TO_SPARE
178 * REMOVING_S-BY_ASS assignment_removed [C111]A23 REPAIRING_SU
179 * REMOVING_S-BY_ASS assignment_removed [C112] IDLE
180 * REMOVING_S-BY_ASS failover_su A52 REMOVING_S-BY_ASS
181 * REMOVING_S-BY_ASS failover_node A52 REMOVING_S-BY_ASS
182 * ACTIVATING_S-BY su_activated [C2]&[C11]A20 ASS_S-BY_TO_SPARE
183 * ACTIVATING_S-BY su_activated [C113]A23 REPAIRING_SU
184 * ACTIVATING_S-BY su_activated [C114] IDLE
185 * ACTIVATING_S-BY failover_su A52 ACTIVATING_S-BY
186 * ACTIVATING_S-BY failover_node A52 ACTIVATING_S-BY
187 * ASS_S-BY_TO_SPARE ha_state_assumed [C115]A23 REPAIRING_SU
188 * ASS_S-BY_TO_SPARE ha_state_assumed [C116] IDLE
189 * ASS_S-BY_TO_SPARE failover_su A52 ASS_S-BY_TO_SPARE
190 * ASS_S-BY_TO_SPARE failover_node A52 ASS_S-BY_TO_SPARE
191 * REPAIRING_SU su_state_chg [C28]A36,A37,A31 ASSIGNING_WL
192 * REPAIRING_SU su_state_chg [C28][C31] IDLE
193 * REPAIRING_SU failover_su A52 REPAIRING_SU
194 * REPAIRING_SU failover_node A52 REPAIRING_SU
195 * ASSIGNING_WL ha_state_assumed [C15] IDLE
196 * ASSIGNING_WL failover_su A52 ASSIGNING_WL
197 * ASSIGNING_WL failover_node A52 ASSIGNING_WL
199 * 1.2 State Description
200 * =====================
201 * IDLE - SG is synchronized and idle. When IDLE state is set,
202 * the oldest deferred event (if any) is recalled.
203 * INSTANTIATING_SUs - INSTANTIATING_SERVICE_UNITS
204 * SG has ordered all contained SUs to instantiate and
205 * waits for the SUs to report a change of their
206 * presence state. SG is also prepared to accept an
207 * order to assign workload in this state.
208 * ASSIGNING_ON_REQ - ASSIGNING_ON_REQUEST
209 * SG has on request assigned workload to all service units
210 * on the requested dependency level and waits for SIs to
211 * indicate that the requested HA-state have been set to the
212 * appropriate components.
213 * TERMINATING_SUSP - TERMINATING_SUSPECTED
214 * SG has cleaned up all components suspected to be
215 * erroneous and waits for the concerned SUs to report a
216 * change of their presence states.
217 * REMOVING_S-BY_ASS - REMOVING_STANDBY_ASSIGNMENTS
218 * This state is only applicable to the n-plus-m redundancy
219 * model. In this state, SG has initiated the removal of
220 * those assignments from standby SUs that do not match the
221 * assignments of error suspected SUs. The reason for this
222 * removal is a preparation for not violating the rule which
223 * says an SU can not have both active and standby assign-
224 * ments simultaneously in the n-plus-m redundancy model.
225 * ACTIVATING_S-BY - ACTIVATING_STANDBY
226 * SG has located all standby SI-assignments in the recovery
227 * scope and ordered the corresponding SI to set the active
228 * HA-state and waits for SI to indicate that the requested
229 * HA-state has been set to the appropriate components.
230 * ASS_S-BY_TO_SPARE - ASSIGNING_STANDBY_TO_SPARE
231 * Current SG is configured with a spare SU. In this state,
232 * SG has requested all SIs to assign the standby HA-state
233 * to the spare SU and waits for the SIs to indicate that
234 * the standby HA-state have been set.
235 * REPAIRING_SU - REPAIRING_SU
236 * In this state SG has initiated instantiation of all SUs
237 * in current recovery scope until the configured preference
238 * of number of instantiated SUs is fulfiled. SG then waits
239 * for the concerned SUs to report a change of presence
241 * ASSIGNING_WL - ASSIGNING_WORKLOAD
242 * In this state SG has initiated the assignment of workload
243 * to all or a subset of its contained SUs and waits for the
244 * concerned SIs to indicated that the requested HA-state
258 * A10 - [foreach SI in the scope]&[foreach SI-assignment with
259 * confirmed-HA-state == ACTIVE]/set requested-ha-state = QUIESCED
260 * A11 - [foreach SI in the scope]/deactivate SI
261 * A12 - [foreach suspected SU]/terminate all components
265 * A16 - [foreach SI in the scope]&[foreach SI-assignment with
266 * confirmed-HA-state == STANDBY]/set requested-ha-state = ACTIVE
267 * A17 - [foreach SI in the scope]/activate SI
302 * 1.4 Composite Guards
303 * ====================
304 * The meaning with these guards is just to save space in the state transition
306 * C100 - [C7]&[!C22]&[C20]
309 * C103 - [C6]&[C21]&[C11]
310 * C104 - [C6]&[C25]&[C26]
311 * C105 - [C6]&(!([C25]|[C26]))&[C24]
312 * C106 - [C6]&(!([C25]|[C26]))&[!C24]&[C11]
314 * C108 - [C6]&(!([C25]|[C26]))&[!C24]&[!C11]&[C9]&[C10]&[!C30]
315 * C109 - [C6]&(!([C25]|[C26]))&[!C24]&[!C11]&[C9]&[C10]&[C30]
316 * C110 - [C27]&[!C24]&[C11]
317 * C111 - [C27]&[!C24]&[!C11]&[C9]&[C10]&[!C30]
318 * C112 - [C27]&[!C24]&[!C11]&[C9]&[C10]&[C30]
319 * C113 - [C2]&[!C11]&[C9]&[C10]&[!C30]
320 * C114 - [C2]&[!C11]&[C9]&[C10]&[C30]
321 * C115 - [C9]&[C10]&[!C30]
322 * C116 - [C9]&[C10]&[C30]
327 * C2 - all SI in the recovery scope
331 * C6 - all supected SUs or components have presence state == UNINSTANTIATED
332 * or presence state == INSTANTIATION_FAILED
333 * C7 - original event == failover node
335 * C9 - original event == failover su
336 * C10 - SaAmfSGAutoRepair == true
337 * C11 - spare SUs exist
338 * C12 - original event == start(all SUs)
339 * C13 - original event == start(node)
341 * C15 - all SI-assignments on current dependency-level have requested-ha-state
342 == confirmed-ha-state or operation failed
347 * C20 - all suspected SUs have presence state == UNINSTANTIATED or
348 presence state == xx_FAILED
349 * C21 - no SI in the scope has an SI-assignment with HA-state == STANDBY
350 * C22 - the concerned entity has ACTIVE HA-state
352 * C24 - at least one SI-assignment related to an SI in the scope has HA-state
354 * C25 - redundancy model == N plus M
355 * C26 - at least one SU has STANDBY assignments for more SIs than those SIs
356 that are within the recovery scope
357 * C27 - no SI-assignment related to SI protected by current SG has requested
359 * C28 - no SU has presence state == INSTANTIATING
361 * C30 - more SUs not needed or the node hosting the SU to repair is disabled.
362 * C31 - no new additional assignments needed or possible
373 LOGSYS_DECLARE_SUBSYS ("AMF", LOG_INFO
);
375 static int assign_si (struct amf_sg
*sg
, int dependency_level
);
376 static void acsm_enter_activating_standby (struct amf_sg
*sg
);
377 static void delete_si_assignments_in_scope (struct amf_sg
*sg
);
378 static void acsm_enter_repairing_su (struct amf_sg
*sg
);
379 static void standby_su_activated_cbfn (
380 struct amf_si_assignment
*si_assignment
, int result
);
381 static void dependent_si_deactivated_cbfn (
382 struct amf_si_assignment
*si_assignment
, int result
);
383 static void dependent_si_deactivated_cbfn2 (struct amf_sg
*sg
);
384 static void assign_si_assumed_cbfn (
385 struct amf_si_assignment
*si_assignment
, int result
);
386 static void acsm_enter_removing_standby_assignments (amf_sg_t
*sg
);
387 static void acsm_enter_assigning_standby_to_spare (amf_sg_t
*sg
);
389 static const char *sg_event_type_text
[] = {
400 typedef struct sg_event
{
401 amf_sg_event_type_t event_type
;
408 /******************************************************************************
409 * Internal (static) utility functions
410 *****************************************************************************/
412 static int is_cluster_start(amf_node_t
*node_to_start
)
414 return node_to_start
== NULL
;
417 static void sg_set_event (amf_sg_event_type_t sg_event_type
,
418 amf_sg_t
*sg
, amf_su_t
*su
, amf_comp_t
*comp
, amf_node_t
* node
,
419 sg_event_t
*sg_event
)
421 sg_event
->event_type
= sg_event_type
;
422 sg_event
->node
= node
;
424 sg_event
->comp
= comp
;
428 static void sg_defer_event (amf_sg_event_type_t event_type
,
429 sg_event_t
*sg_event
)
431 ENTER("Defered event = %d", event_type
);
432 amf_fifo_put (event_type
, &sg_event
->sg
->deferred_events
,
437 static void sg_recall_deferred_events (amf_sg_t
*sg
)
441 ENTER ("%s", sg
->name
.value
);
442 if (amf_fifo_get (&sg
->deferred_events
, &sg_event
)) {
443 switch (sg_event
.event_type
) {
444 case SG_FAILOVER_SU_EV
:
445 amf_sg_failover_su_req (sg_event
.sg
,
446 sg_event
.su
, sg_event
.node
);
448 case SG_FAILOVER_NODE_EV
:
449 amf_sg_failover_node_req (sg_event
.sg
,
452 case SG_FAILOVER_COMP_EV
:
453 case SG_SWITCH_OVER_NODE_EV
:
455 case SG_AUTO_ADJUST_EV
:
457 dprintf("event_type = %d", sg_event
.event_type
);
463 static void timer_function_sg_recall_deferred_events (void *data
)
465 amf_sg_t
*sg
= (amf_sg_t
*)data
;
468 sg_recall_deferred_events (sg
);
471 static void acsm_enter_idle (amf_sg_t
*sg
)
475 ENTER ("sg: %s state: %d", sg
->name
.value
, sg
->avail_state
);
477 sg
->avail_state
= SG_AC_Idle
;
478 if (sg
->recovery_scope
.event_type
!= 0) {
479 switch (sg
->recovery_scope
.event_type
) {
480 case SG_FAILOVER_SU_EV
:
481 assert (sg
->recovery_scope
.sus
[0] != NULL
);
482 amf_su_dn_make (sg
->recovery_scope
.sus
[0], &dn
);
485 "'%s' %s recovery action finished",
487 sg_event_type_text
[sg
->recovery_scope
.event_type
]);
490 case SG_FAILOVER_NODE_EV
:
491 amf_node_sg_failed_over (
492 sg
->recovery_scope
.node
, sg
);
495 "'%s for %s' recovery action finished",
496 sg_event_type_text
[sg
->recovery_scope
.event_type
],
500 amf_application_sg_started (sg
->application
,
503 case SG_ASSIGN_SI_EV
:
504 log_printf (LOG_NOTICE
, "All SI assigned");
509 "'%s' recovery action finished",
510 sg_event_type_text
[sg
->recovery_scope
.event_type
]);
515 if (sg
->recovery_scope
.sus
!= NULL
) {
516 free ((void *)sg
->recovery_scope
.sus
);
518 if (sg
->recovery_scope
.sis
!= NULL
) {
519 free ((void *)sg
->recovery_scope
.sis
);
521 memset (&sg
->recovery_scope
, 0, sizeof (struct sg_recovery_scope
));
522 sg
->node_to_start
= NULL
;
524 amf_call_function_asynchronous (
525 timer_function_sg_recall_deferred_events
, sg
);
528 static int su_instantiated_count (struct amf_sg
*sg
)
533 for (su
= sg
->su_head
; su
!= NULL
; su
= su
->next
) {
534 if (su
->saAmfSUPresenceState
== SA_AMF_PRESENCE_INSTANTIATED
)
541 static int has_any_su_in_scope_active_workload (struct amf_sg
*sg
)
543 struct amf_su
**sus
= sg
->recovery_scope
.sus
;
544 struct amf_si_assignment
*si_assignment
;
546 while (*sus
!= NULL
) {
547 si_assignment
= amf_su_get_next_si_assignment (*sus
, NULL
);
548 while (si_assignment
!= NULL
) {
549 if (si_assignment
->saAmfSISUHAState
!=
553 si_assignment
= amf_su_get_next_si_assignment (
554 *sus
, si_assignment
);
556 if (si_assignment
!= NULL
) {
561 return(*sus
== NULL
);
564 static int is_any_si_in_scope_assigned_standby (struct amf_sg
*sg
)
566 struct amf_si
**sis
= sg
->recovery_scope
.sis
;
567 struct amf_si_assignment
*si_assignment
;
570 * Check if there is any si in the scope which has no
571 * active assignment and at least one standby assignment.
573 while (*sis
!= NULL
) {
574 si_assignment
= (*sis
)->assigned_sis
;
575 while (si_assignment
!= NULL
) {
576 if (si_assignment
->saAmfSISUHAState
==
580 si_assignment
= si_assignment
->next
;
582 if (si_assignment
== NULL
) {
583 /* There is no ACTIVE assignment ..*/
584 si_assignment
= (*sis
)->assigned_sis
;
585 while (si_assignment
!= NULL
) {
586 if (si_assignment
->saAmfSISUHAState
==
590 si_assignment
= si_assignment
->next
;
592 if (si_assignment
!= NULL
) {
593 /* .. and one STANDBY assignment*/
599 return(*sis
!= NULL
);
603 static void acsm_enter_terminating_suspected (struct amf_sg
*sg
)
605 struct amf_su
**sus
= sg
->recovery_scope
.sus
;
606 ENTER("%s",sg
->name
.value
);
607 sg
->avail_state
= SG_AC_TerminatingSuspected
;
609 * Terminate suspected SU(s)
612 amf_su_terminate (*sus
);
617 static inline int su_presense_state_is_ored (amf_su_t
*su
,
618 SaAmfPresenceStateT state1
,SaAmfPresenceStateT state2
,
619 SaAmfPresenceStateT state3
)
621 return(su
->saAmfSUPresenceState
== state1
|| su
->saAmfSUPresenceState
==
622 state2
|| su
->saAmfSUPresenceState
== state3
) ? 1 : 0;
625 static inline int su_presense_state_is_not (amf_su_t
*su
,
626 SaAmfPresenceStateT state1
,SaAmfPresenceStateT state2
,
627 SaAmfPresenceStateT state3
)
629 return(su
->saAmfSUPresenceState
!= state1
&& su
->saAmfSUPresenceState
!=
630 state2
&& su
->saAmfSUPresenceState
!= state3
) ? 1 : 0;
634 static void timer_function_dependent_si_deactivated2 (void *data
)
636 amf_sg_t
*sg
= (amf_sg_t
*)data
;
639 dependent_si_deactivated_cbfn2 (sg
);
643 static struct amf_si
*si_get_dependent (struct amf_si
*si
)
645 struct amf_si
*tmp_si
= NULL
;
647 if (si
->depends_on
!= NULL
) {
651 if (si
->depends_on
->name
.length
< SA_MAX_NAME_LENGTH
) {
652 si
->depends_on
->name
.value
[si
->depends_on
->name
.length
] = '\0';
655 is_match
= sa_amf_grep ((char*)si
->depends_on
->name
.value
,
656 "safDepend=.*,safSi=(.*),safApp=.*",
660 tmp_si
= amf_si_find (si
->application
, (char*)res_arr
[1].value
);
662 log_printf (LOG_LEVEL_ERROR
, "distinguished name for "
663 "amf_si_depedency failed\n");
664 openais_exit_error (AIS_DONE_FATAL_ERR
);
671 static struct amf_si
*amf_dependent_get_next (struct amf_si
*si
,
672 struct amf_si
*si_iter
)
674 struct amf_si
*tmp_si
;
675 struct amf_application
*application
;
677 if (si_iter
== NULL
) {
678 assert(amf_cluster
!= NULL
);
679 application
= amf_cluster
->application_head
;
680 assert(application
!= NULL
);
681 tmp_si
= application
->si_head
;
683 tmp_si
= si_iter
->next
;
684 if (tmp_si
== NULL
) {
685 application
= si
->application
->next
;
686 if (application
== NULL
) {
692 for (; tmp_si
!= NULL
; tmp_si
= tmp_si
->next
) {
693 struct amf_si
*depends_on_si
= si_get_dependent (tmp_si
);
694 while (depends_on_si
!= NULL
) {
695 if (depends_on_si
== si
) {
698 depends_on_si
= depends_on_si
->next
;
706 static void acsm_enter_deactivating_dependent_workload (amf_sg_t
*sg
)
708 struct amf_si
**sis
= sg
->recovery_scope
.sis
;
709 struct amf_si_assignment
*si_assignment
;
710 int callback_pending
= 0;
712 sg
->avail_state
= SG_AC_DeactivatingDependantWorkload
;
714 ENTER("'%s'",sg
->name
.value
);
716 * For each SI in the recovery scope, find all active
717 * assignments and request them to be deactivated.
719 while (*sis
!= NULL
) {
720 struct amf_si
*dependent_si
;
721 struct amf_si
*si
= *sis
;
722 si_assignment
= si
->assigned_sis
;
723 dependent_si
= amf_dependent_get_next (si
, NULL
);
725 while (dependent_si
!= NULL
) {
726 si_assignment
= dependent_si
->assigned_sis
;
728 while (si_assignment
!= NULL
) {
730 if (si_assignment
->saAmfSISUHAState
==
732 si_assignment
->requested_ha_state
=
734 callback_pending
= 1;
735 amf_si_ha_state_assume (
737 dependent_si_deactivated_cbfn
);
739 si_assignment
= si_assignment
->next
;
741 dependent_si
= amf_dependent_get_next (si
, dependent_si
);
746 if (callback_pending
== 0) {
747 static poll_timer_handle dependent_si_deactivated_handle
;
749 poll_timer_add (aisexec_poll_handle
, 0, sg
,
750 timer_function_dependent_si_deactivated2
,
751 &dependent_si_deactivated_handle
);
755 * Enter function for state SG_AC_ActivatingStandby. It activates
756 * one STANDBY assignment for each SI in the recovery scope.
759 static void acsm_enter_activating_standby (struct amf_sg
*sg
)
761 struct amf_si
**sis
= sg
->recovery_scope
.sis
;
762 struct amf_si_assignment
*si_assignment
;
763 int is_no_standby_activated
= 1;
765 ENTER("'%s'",sg
->name
.value
);
766 sg
->avail_state
= SG_AC_ActivatingStandby
;
769 * For each SI in the recovery scope, find one standby
770 * SI assignment and activate it.
772 while (*sis
!= NULL
) {
773 si_assignment
= (*sis
)->assigned_sis
;
774 while (si_assignment
!= NULL
) {
775 if (si_assignment
->saAmfSISUHAState
==
777 si_assignment
->requested_ha_state
=
779 amf_si_ha_state_assume (
780 si_assignment
, standby_su_activated_cbfn
);
781 is_no_standby_activated
= 0;
784 si_assignment
= si_assignment
->next
;
789 if (is_no_standby_activated
) {
791 acsm_enter_assigning_standby_to_spare (sg
);
795 static void acsm_enter_repairing_su (struct amf_sg
*sg
)
797 struct amf_su
**sus
= sg
->recovery_scope
.sus
;
798 int is_any_su_instantiated
= 0;
799 const int PERFORMS_INSTANTIATING
= 1;
801 ENTER("'%s'",sg
->name
.value
);
802 sg
->avail_state
= SG_AC_ReparingSu
;
804 * Instantiate SUs in current recovery scope until the configured
805 * preference is fulfiled.
807 while (*sus
!= NULL
) {
808 if (su_instantiated_count ((*sus
)->sg
) <
809 (*sus
)->sg
->saAmfSGNumPrefInserviceSUs
) {
810 struct amf_node
*node
=
811 amf_node_find(&((*sus
)->saAmfSUHostedByNode
));
813 log_printf (LOG_LEVEL_ERROR
,
814 "Su to recover not hosted on any node\n");
815 openais_exit_error (AIS_DONE_FATAL_ERR
);
817 if (node
->saAmfNodeOperState
== SA_AMF_OPERATIONAL_ENABLED
) {
818 /* node is synchronized */
820 if (amf_su_instantiate ((*sus
)) == PERFORMS_INSTANTIATING
) {
821 is_any_su_instantiated
= 1;
828 if (is_any_su_instantiated
== 0) {
829 acsm_enter_idle (sg
);
833 static inline void remove_all_suspected_sus (amf_sg_t
*sg
)
837 for (su
= sg
->su_head
; su
!= NULL
; su
=su
->next
) {
839 amf_comp_t
*component
;
841 for (component
= su
->comp_head
; component
!= NULL
;
842 component
= component
->next
) {
844 amf_comp_error_suspected_clear (component
);
848 static int is_all_si_assigned (amf_sg_t
*sg
)
850 struct amf_si_assignment
*si_assignment
;
851 int si_assignment_cnt
= 0;
852 int confirmed_assignments
= 0;
855 for (si
= sg
->application
->si_head
; si
!= NULL
; si
= si
->next
) {
856 if (name_match (&si
->saAmfSIProtectedbySG
, &sg
->name
)) {
858 for (si_assignment
= si
->assigned_sis
;
859 si_assignment
!= NULL
;
860 si_assignment
= si_assignment
->next
) {
863 if (si_assignment
->requested_ha_state
==
864 si_assignment
->saAmfSISUHAState
) {
866 confirmed_assignments
++;
871 return (confirmed_assignments
== si_assignment_cnt
);
875 * Inquire if SI is assigned to SU
881 static int is_si_assigned_to_su (amf_si_t
*si
, amf_su_t
*su
)
883 amf_si_assignment_t
*si_assignment
= 0;
884 int si_assignment_assigned_to_su
= 0;
886 for (si_assignment
= si
->assigned_sis
; si_assignment
!= NULL
;
887 si_assignment
= si_assignment
->next
) {
889 if (si_assignment
->su
== su
) {
890 si_assignment_assigned_to_su
= 1;
894 return si_assignment_assigned_to_su
;
898 * Inquire if SU is a spare.
904 static int is_spare_su (amf_sg_t
*sg
, amf_su_t
*su
)
908 for (si
= sg
->application
->si_head
; si
!= NULL
; si
= si
->next
) {
909 if(name_match(&sg
->name
, &si
->saAmfSIProtectedbySG
)) {
910 if (is_si_assigned_to_su (si
, su
)) {
917 return (spare_su
&& su
->saAmfSUPresenceState
==
918 SA_AMF_PRESENCE_INSTANTIATED
);
921 * Inqure if it is any spare SUs covered by SG
926 static int is_spare_sus (amf_sg_t
*sg
)
931 for (su
= sg
->su_head
; su
!= NULL
; su
= su
->next
) {
933 if (is_spare_su(sg
, su
)) {
942 * Provide standby assignments for the spare SUs in SG
945 static void assume_standby_si_assignment_for_spare_sus (amf_sg_t
*sg
)
953 * Enter the AssigningStandbyToSpare state.
956 static void acsm_enter_assigning_standby_to_spare (amf_sg_t
*sg
)
958 ENTER("%s", sg
->name
.value
);
960 sg
->avail_state
= SG_AC_AssigningStandbyToSpare
;
961 if (is_spare_sus (sg
)) {
962 assume_standby_si_assignment_for_spare_sus (sg
);
964 switch (sg
->recovery_scope
.event_type
) {
965 case SG_FAILOVER_NODE_EV
:
966 acsm_enter_idle (sg
);
968 case SG_FAILOVER_SU_EV
:
969 acsm_enter_repairing_su (sg
);
972 dprintf("event_type %d",sg
->recovery_scope
.event_type
);
980 * Checks if the si pointed out is already in the scope.
984 static int is_si_in_scope(struct amf_sg
*sg
, struct amf_si
*si
)
986 struct amf_si
**tmp_sis
= sg
->recovery_scope
.sis
;
988 while (*tmp_sis
!= NULL
) {
989 if (*tmp_sis
== si
) {
994 return(*tmp_sis
== si
);
998 * Adds the si pointed out to the scope.
1002 static void add_si_to_scope ( struct amf_sg
*sg
, struct amf_si
*si
)
1004 int number_of_si
= 2; /* It shall be at least two */
1005 struct amf_si
**tmp_sis
= sg
->recovery_scope
.sis
;
1007 ENTER ("'%s'", si
->name
.value
);
1009 while (*tmp_sis
!= NULL
) {
1014 sg
->recovery_scope
.sis
= (struct amf_si
**)
1015 realloc((void *)sg
->recovery_scope
.sis
,
1016 sizeof (struct amf_si
*)*number_of_si
);
1017 assert (sg
->recovery_scope
.sis
!= NULL
);
1019 tmp_sis
= sg
->recovery_scope
.sis
;
1020 while (*tmp_sis
!= NULL
) {
1025 *(++tmp_sis
) = NULL
;
1029 * Adds the ssu pointed out to the scope.
1033 static void add_su_to_scope (struct amf_sg
*sg
, struct amf_su
*su
)
1035 int number_of_su
= 2; /* It shall be at least two */
1036 struct amf_su
**tmp_sus
= sg
->recovery_scope
.sus
;
1038 ENTER ("'%s'", su
->name
.value
);
1039 while (*tmp_sus
!= NULL
) {
1043 sg
->recovery_scope
.sus
= (struct amf_su
**)
1044 realloc((void *)sg
->recovery_scope
.sus
,
1045 sizeof (struct amf_su
*)*number_of_su
);
1046 assert (sg
->recovery_scope
.sus
!= NULL
);
1048 tmp_sus
= sg
->recovery_scope
.sus
;
1049 while (*tmp_sus
!= NULL
) {
1054 *(++tmp_sus
) = NULL
;
1058 * Set recovery scope for failover SU.
1063 static void set_scope_for_failover_su (struct amf_sg
*sg
, struct amf_su
*su
)
1065 struct amf_si_assignment
*si_assignment
;
1066 struct amf_si
**sis
;
1067 struct amf_su
**sus
;
1070 sg
->recovery_scope
.event_type
= SG_FAILOVER_SU_EV
;
1071 sg
->recovery_scope
.node
= NULL
;
1072 sg
->recovery_scope
.comp
= NULL
;
1073 sg
->recovery_scope
.sus
= (struct amf_su
**)
1074 calloc (2, sizeof (struct amf_su
*));
1075 sg
->recovery_scope
.sis
= (struct amf_si
**)
1076 calloc (1, sizeof (struct amf_si
*));
1078 assert ((sg
->recovery_scope
.sus
!= NULL
) &&
1079 (sg
->recovery_scope
.sis
!= NULL
));
1080 sg
->recovery_scope
.sus
[0] = su
;
1082 amf_su_dn_make (sg
->recovery_scope
.sus
[0], &dn
);
1084 LOG_NOTICE
, "'%s' for %s recovery action started",
1085 sg_event_type_text
[sg
->recovery_scope
.event_type
],
1088 si_assignment
= amf_su_get_next_si_assignment (su
, NULL
);
1089 while (si_assignment
!= NULL
) {
1090 if (is_si_in_scope(sg
, si_assignment
->si
) == 0) {
1091 add_si_to_scope(sg
,si_assignment
->si
);
1093 si_assignment
= amf_su_get_next_si_assignment (su
, si_assignment
);
1096 sus
= sg
->recovery_scope
.sus
;
1097 dprintf("The following sus are within the scope:\n");
1098 while (*sus
!= NULL
) {
1099 dprintf("%s\n", (*sus
)->name
.value
);
1102 sis
= sg
->recovery_scope
.sis
;
1103 dprintf("The following sis are within the scope:\n");
1104 while (*sis
!= NULL
) {
1105 dprintf("%s\n", (*sis
)->name
.value
);
1110 static void set_scope_for_failover_node (struct amf_sg
*sg
, struct amf_node
*node
)
1112 struct amf_si_assignment
*si_assignment
;
1113 struct amf_si
**sis
;
1114 struct amf_su
**sus
;
1117 ENTER ("'%s'", node
->name
.value
);
1118 sg
->recovery_scope
.event_type
= SG_FAILOVER_NODE_EV
;
1119 sg
->recovery_scope
.node
= node
;
1120 sg
->recovery_scope
.comp
= NULL
;
1121 sg
->recovery_scope
.sus
= (struct amf_su
**)
1122 calloc (1, sizeof (struct amf_su
*));
1123 sg
->recovery_scope
.sis
= (struct amf_si
**)
1124 calloc (1, sizeof (struct amf_si
*));
1127 LOG_NOTICE
, "'%s' for node %s recovery action started",
1128 sg_event_type_text
[sg
->recovery_scope
.event_type
],
1131 assert ((sg
->recovery_scope
.sus
!= NULL
) &&
1132 (sg
->recovery_scope
.sis
!= NULL
));
1133 for (su
= sg
->su_head
; su
!= NULL
; su
= su
->next
) {
1134 if (name_match (&node
->name
, &su
->saAmfSUHostedByNode
)) {
1135 add_su_to_scope (sg
, su
);
1139 sus
= sg
->recovery_scope
.sus
;
1142 si_assignment
= amf_su_get_next_si_assignment (su
, NULL
);
1143 while (si_assignment
!= NULL
) {
1144 if (is_si_in_scope(sg
, si_assignment
->si
) == 0) {
1145 add_si_to_scope(sg
, si_assignment
->si
);
1147 si_assignment
= amf_su_get_next_si_assignment (
1153 sus
= sg
->recovery_scope
.sus
;
1154 dprintf("The following sus are within the scope:\n");
1155 while (*sus
!= NULL
) {
1156 dprintf("%s\n", (*sus
)->name
.value
);
1159 sis
= sg
->recovery_scope
.sis
;
1160 dprintf("The following sis are within the scope:\n");
1161 while (*sis
!= NULL
) {
1162 dprintf("%s\n", (*sis
)->name
.value
);
1167 static void delete_si_assignment (amf_si_assignment_t
*si_assignment
)
1170 amf_si_assignment_t
*si_assignment_tmp
;
1171 amf_si_assignment_t
**prev
= &si_assignment
->si
->assigned_sis
;
1173 for (csi
= si_assignment
->si
->csi_head
; csi
!= NULL
; csi
= csi
->next
) {
1174 amf_csi_delete_assignments (csi
, si_assignment
->su
);
1177 for (si_assignment_tmp
= si_assignment
->si
->assigned_sis
;
1178 si_assignment_tmp
!= NULL
;
1179 si_assignment_tmp
= si_assignment_tmp
->next
) {
1181 if (si_assignment_tmp
== si_assignment
) {
1183 amf_si_assignment_t
*to_be_removed
= si_assignment_tmp
;
1184 *prev
= si_assignment_tmp
->next
;
1185 dprintf ("SI assignment %s unlinked",
1186 to_be_removed
->name
.value
);
1187 free (to_be_removed
);
1189 prev
= &si_assignment_tmp
->next
;
1195 * Delete all SI assignments and all CSI assignments
1196 * by requesting all contained components.
1199 static void delete_si_assignments (struct amf_su
*su
)
1201 struct amf_csi
*csi
;
1203 struct amf_si_assignment
*si_assignment
;
1204 struct amf_si_assignment
**prev
;
1206 ENTER ("'%s'", su
->name
.value
);
1208 for (si
= su
->sg
->application
->si_head
; si
!= NULL
; si
= si
->next
) {
1210 prev
= &si
->assigned_sis
;
1212 if (!name_match (&si
->saAmfSIProtectedbySG
, &su
->sg
->name
)) {
1216 for (csi
= si
->csi_head
; csi
!= NULL
; csi
= csi
->next
) {
1217 amf_csi_delete_assignments (csi
, su
);
1220 for (si_assignment
= si
->assigned_sis
; si_assignment
!= NULL
;
1221 si_assignment
= si_assignment
->next
) {
1222 if (si_assignment
->su
== su
) {
1223 struct amf_si_assignment
*tmp
= si_assignment
;
1224 *prev
= si_assignment
->next
;
1225 dprintf ("SI assignment %s unlinked", tmp
->name
.value
);
1228 prev
= &si_assignment
->next
;
1235 * Delete all SI assignments and all CSI assignments in current
1239 static void delete_si_assignments_in_scope (struct amf_sg
*sg
)
1241 struct amf_su
**sus
= sg
->recovery_scope
.sus
;
1243 while (*sus
!= NULL
) {
1244 delete_si_assignments (*sus
);
1250 * Given an SI, find and return the SU assigned as standby
1255 static amf_su_t
*find_standby_su (amf_si_t
*si
)
1257 amf_si_assignment_t
*si_assignment
;
1258 amf_su_t
*standby_su
= NULL
;
1260 si_assignment
= si
->assigned_sis
;
1261 while (si_assignment
!= NULL
) {
1262 if (si_assignment
->saAmfSISUHAState
== SA_AMF_HA_STANDBY
) {
1263 standby_su
= si_assignment
->su
;
1266 si_assignment
= si_assignment
->next
;
1272 static int no_si_assignment_is_requested_to_be_removed (amf_sg_t
*sg
)
1275 int no_to_be_removed
= 1;
1277 for (si
= sg
->application
->si_head
; si
!= NULL
; si
= si
->next
) {
1279 if (name_match (&si
->saAmfSIProtectedbySG
, &sg
->name
)) {
1280 amf_si_assignment_t
*si_assignment
= 0;
1281 for (si_assignment
= si
->assigned_sis
; si_assignment
!= NULL
;
1282 si_assignment
= si_assignment
->next
) {
1283 if (si_assignment
->requested_ha_state
==
1284 USR_AMF_HA_STATE_REMOVED
) {
1286 no_to_be_removed
= 0;
1293 return no_to_be_removed
;
1296 static void removed_si_assignment_callback_fn (void *si_assignment_in
)
1298 amf_si_assignment_t
*si_assignment
= si_assignment_in
;
1301 delete_si_assignment (si_assignment
);
1304 * if all si assignments are remove then change state
1306 if (no_si_assignment_is_requested_to_be_removed (si_assignment
->su
->sg
)) {
1307 acsm_enter_activating_standby (si_assignment
->su
->sg
);
1315 * @return int, number of removed SI assignments
1317 static int remove_standby_si_assignments (amf_sg_t
*sg
)
1319 struct amf_si
**sis
= sg
->recovery_scope
.sis
;
1320 struct amf_si_assignment
*si_assignment
;
1321 amf_su_t
*standby_su
;
1324 ENTER("'%s'", sg
->name
.value
);
1327 * For each SI in the recovery scope, find a standby
1328 * SU, then remove all 'standby SI assignment' not in
1329 * the recovery scope.
1331 while (*sis
!= NULL
) {
1332 standby_su
= find_standby_su (*sis
);
1333 if (standby_su
!= NULL
) {
1334 si_assignment
= amf_su_get_next_si_assignment (standby_su
, NULL
);
1335 while (si_assignment
!= NULL
) {
1338 int in_recovery_scope
;
1340 for (sia
= sg
->recovery_scope
.sis
, in_recovery_scope
= 0;
1341 *sia
!= NULL
; sia
++) {
1342 if (name_match (&si_assignment
->si
->name
, &(*sia
)->name
)) {
1343 in_recovery_scope
= 1;
1348 * The si_assignment found with standby hastate is not in the
1349 * recovery scope. The found si_assignment will then be
1350 * requested to be removed once.
1352 if (!in_recovery_scope
&&
1353 si_assignment
->requested_ha_state
!=
1354 USR_AMF_HA_STATE_REMOVED
) {
1356 amf_si_assignment_remove (si_assignment
,
1357 removed_si_assignment_callback_fn
);
1360 si_assignment
= amf_su_get_next_si_assignment (standby_su
,
1371 * Entry function for state 'removing standby assignments'
1374 static void acsm_enter_removing_standby_assignments (amf_sg_t
*sg
)
1376 ENTER("SG: %s", sg
->name
.value
);
1377 sg
->avail_state
= SG_AC_RemovingStandbyAssignments
;
1379 if (sg
->saAmfSGRedundancyModel
== SA_AMF_NPM_REDUNDANCY_MODEL
) {
1380 if (!remove_standby_si_assignments (sg
)) {
1381 acsm_enter_activating_standby (sg
);
1386 static inline int div_round (int a
, int b
)
1397 static int no_su_has_presence_state (
1398 struct amf_sg
*sg
, struct amf_node
*node_to_start
,
1399 SaAmfPresenceStateT state
)
1402 int no_su_has_presence_state
= 1;
1404 for (su
= sg
->su_head
; su
!= NULL
; su
= su
->next
) {
1406 if (su
->saAmfSUPresenceState
== state
) {
1407 if (node_to_start
== NULL
) {
1408 no_su_has_presence_state
= 0;
1411 if (name_match(&node_to_start
->name
,
1412 &su
->saAmfSUHostedByNode
)) {
1413 no_su_has_presence_state
= 0;
1420 return no_su_has_presence_state
;
1424 static int all_su_in_scope_has_presence_state (
1425 struct amf_sg
*sg
, SaAmfPresenceStateT state
)
1427 struct amf_su
**sus
= sg
->recovery_scope
.sus
;
1429 while (*sus
!= NULL
) {
1430 if ((*sus
)->saAmfSUPresenceState
!= state
) {
1435 return(*sus
== NULL
);
1438 static int all_su_in_scope_has_either_two_presence_state (
1440 SaAmfPresenceStateT state1
,
1441 SaAmfPresenceStateT state2
)
1443 struct amf_su
**sus
= sg
->recovery_scope
.sus
;
1445 while (*sus
!= NULL
) {
1446 if (!((*sus
)->saAmfSUPresenceState
== state1
||
1447 (*sus
)->saAmfSUPresenceState
== state2
)) {
1452 return (*sus
== NULL
);
1456 static int all_su_in_scope_has_either_of_three_presence_state (amf_sg_t
*sg
,
1457 SaAmfPresenceStateT state1
, SaAmfPresenceStateT state2
,
1458 SaAmfPresenceStateT state3
)
1460 struct amf_su
**sus
= sg
->recovery_scope
.sus
;
1462 while (*sus
!= NULL
) {
1463 if (!((*sus
)->saAmfSUPresenceState
== state1
||
1464 (*sus
)->saAmfSUPresenceState
== state2
||
1465 (*sus
)->saAmfSUPresenceState
== state3
)) {
1470 return (*sus
== NULL
);
1476 * Get number of SIs protected by the specified SG.
1481 static int sg_si_count_get (struct amf_sg
*sg
)
1486 for (si
= sg
->application
->si_head
; si
!= NULL
; si
= si
->next
) {
1487 if (name_match (&si
->saAmfSIProtectedbySG
, &sg
->name
)) {
1494 static int amf_si_get_saAmfSINumReqActiveAssignments(struct amf_si
*si
)
1496 struct amf_si_assignment
*si_assignment
= si
->assigned_sis
;
1497 int number_of_req_active_assignments
= 0;
1499 for (; si_assignment
!= NULL
; si_assignment
= si_assignment
->next
) {
1501 if (si_assignment
->requested_ha_state
== SA_AMF_HA_ACTIVE
) {
1502 number_of_req_active_assignments
++;
1505 return number_of_req_active_assignments
;
1508 static int amf_si_get_saAmfSINumReqStandbyAssignments(struct amf_si
*si
)
1510 struct amf_si_assignment
*si_assignment
= si
->assigned_sis
;
1511 int number_of_req_active_assignments
= 0;
1513 for (; si_assignment
!= NULL
; si_assignment
= si_assignment
->next
) {
1514 if (si_assignment
->requested_ha_state
== SA_AMF_HA_STANDBY
) {
1515 number_of_req_active_assignments
++;
1518 return number_of_req_active_assignments
;
1521 static int sg_assign_active_nplusm (struct amf_sg
*sg
, int su_active_assign
)
1526 int assign_to_su
= 0;
1527 int total_assigned
= 0;
1530 int su_left_to_assign
= su_active_assign
;
1531 ENTER("SG: %s", sg
->name
.value
);
1533 si_total
= sg_si_count_get (sg
);
1535 assign_to_su
= div_round (si_left
, su_active_assign
);
1536 if (assign_to_su
> sg
->saAmfSGMaxActiveSIsperSUs
) {
1537 assign_to_su
= sg
->saAmfSGMaxActiveSIsperSUs
;
1541 while (su
!= NULL
&& su_left_to_assign
> 0) {
1542 if (amf_su_get_saAmfSUReadinessState (su
) !=
1543 SA_AMF_READINESS_IN_SERVICE
||
1544 amf_su_get_saAmfSUNumCurrActiveSIs (su
) == assign_to_su
||
1545 amf_su_get_saAmfSUNumCurrStandbySIs (su
) > 0) {
1548 continue; /* Not in service */
1551 si
= sg
->application
->si_head
;
1553 assign_to_su
= div_round (si_left
, su_left_to_assign
);
1554 if (assign_to_su
> sg
->saAmfSGMaxActiveSIsperSUs
) {
1555 assign_to_su
= sg
->saAmfSGMaxActiveSIsperSUs
;
1557 while (si
!= NULL
) {
1558 if (name_match (&si
->saAmfSIProtectedbySG
, &sg
->name
) &&
1559 assigned
< assign_to_su
&&
1560 amf_si_get_saAmfSINumReqActiveAssignments(si
) == 0) {
1562 total_assigned
+= 1;
1563 amf_su_assign_si (su
, si
, SA_AMF_HA_ACTIVE
);
1569 su_left_to_assign
-= 1;
1570 si_left
-= assigned
;
1571 dprintf (" su_left_to_assign =%d, si_left=%d\n",
1572 su_left_to_assign
, si_left
);
1575 assert (total_assigned
<= si_total
);
1576 if (total_assigned
== 0) {
1577 dprintf ("Info: No SIs assigned");
1580 return total_assigned
;
1583 static int sg_assign_standby_nplusm (struct amf_sg
*sg
, int su_standby_assign
)
1588 int assign_to_su
= 0;
1589 int total_assigned
= 0;
1592 int su_left_to_assign
= su_standby_assign
;
1594 ENTER ("'%s'", sg
->name
.value
);
1596 if (su_standby_assign
== 0) {
1599 si_total
= sg_si_count_get (sg
);
1601 assign_to_su
= div_round (si_left
, su_standby_assign
);
1602 if (assign_to_su
> sg
->saAmfSGMaxStandbySIsperSUs
) {
1603 assign_to_su
= sg
->saAmfSGMaxStandbySIsperSUs
;
1607 while (su
!= NULL
&& su_left_to_assign
> 0) {
1608 if (amf_su_get_saAmfSUReadinessState (su
) !=
1609 SA_AMF_READINESS_IN_SERVICE
||
1610 amf_su_get_saAmfSUNumCurrActiveSIs (su
) > 0 ||
1611 amf_su_get_saAmfSUNumCurrStandbySIs (su
) ==
1615 continue; /* Not available for assignment */
1618 si
= sg
->application
->si_head
;
1620 assign_to_su
= div_round (si_left
, su_left_to_assign
);
1621 if (assign_to_su
> sg
->saAmfSGMaxStandbySIsperSUs
) {
1622 assign_to_su
= sg
->saAmfSGMaxStandbySIsperSUs
;
1624 while (si
!= NULL
) {
1625 if (name_match (&si
->saAmfSIProtectedbySG
, &sg
->name
) &&
1626 assigned
< assign_to_su
&&
1627 amf_si_get_saAmfSINumReqStandbyAssignments (si
) == 0) {
1629 total_assigned
+= 1;
1630 amf_su_assign_si (su
, si
, SA_AMF_HA_STANDBY
);
1634 su_left_to_assign
-= 1;
1635 si_left
-= assigned
;
1636 dprintf (" su_left_to_assign =%d, si_left=%d\n",
1637 su_left_to_assign
, si_left
);
1642 assert (total_assigned
<= si_total
);
1643 if (total_assigned
== 0) {
1644 dprintf ("Info: No SIs assigned!");
1647 return total_assigned
;
1650 static int su_inservice_count_get (struct amf_sg
*sg
)
1655 for (su
= sg
->su_head
; su
!= NULL
; su
= su
->next
) {
1656 if (amf_su_get_saAmfSUReadinessState (su
) ==
1657 SA_AMF_READINESS_IN_SERVICE
) {
1665 static int su_active_out_of_service_count_get (amf_sg_t
*sg
)
1667 int active_out_of_service_count
= 0;
1669 for (su
= sg
->su_head
; su
!= NULL
; su
= su
->next
) {
1670 amf_si_assignment_t
*si_assignment
;
1671 si_assignment
= amf_su_get_next_si_assignment (su
, NULL
);
1672 while (si_assignment
!= NULL
) {
1673 if ((si_assignment
->saAmfSISUHAState
== SA_AMF_HA_ACTIVE
) &&
1674 (amf_su_get_saAmfSUReadinessState (su
) ==
1675 SA_AMF_READINESS_OUT_OF_SERVICE
)) {
1676 active_out_of_service_count
+= 1;
1678 si_assignment
= amf_su_get_next_si_assignment (su
, si_assignment
);
1681 return active_out_of_service_count
;
1685 static int su_standby_out_of_service_count_get (amf_sg_t
*sg
)
1687 int active_out_of_service_count
= 0;
1689 for (su
= sg
->su_head
; su
!= NULL
; su
= su
->next
) {
1690 amf_si_assignment_t
*si_assignment
;
1691 si_assignment
= amf_su_get_next_si_assignment (su
, NULL
);
1692 while (si_assignment
!= NULL
) {
1693 if ((si_assignment
->saAmfSISUHAState
== SA_AMF_HA_STANDBY
) &&
1694 (amf_su_get_saAmfSUReadinessState (su
) ==
1695 SA_AMF_READINESS_OUT_OF_SERVICE
)) {
1696 active_out_of_service_count
+= 1;
1698 si_assignment
= amf_su_get_next_si_assignment (su
, si_assignment
);
1701 return active_out_of_service_count
;
1705 * This function calculates the number of active and standby assignments that
1706 * shall be done according to what is configured and the number of in-service
1707 * SUs available. This function leaves possible existing assignments as they are
1708 * but possibly adds new assignments. This function also initiates the transfer
1709 * of the calculated assignments to the SUs that shall execute them.
1711 * TODO: dependency_level not used, hard coded
1713 * @param dependency_level
1714 * @return - the sum of assignments initiated
1716 static int assign_si (struct amf_sg
*sg
, int dependency_level
)
1718 int active_sus_needed
= 0;
1719 int standby_sus_needed
= 0;
1720 int inservice_count
;
1721 int su_active_assign
;
1722 int su_standby_assign
;
1723 int su_spare_assign
;
1725 int active_out_of_service
= 0;
1726 int standby_out_of_service
= 0;
1727 ENTER ("'%s'", sg
->name
.value
);
1730 * Phase 1: Calculate assignments and create all runtime objects in
1731 * information model. Do not do the actual assignment, done in
1736 * Calculate number of SUs to assign to active or standby state
1738 inservice_count
= su_inservice_count_get (sg
);
1739 active_out_of_service
= su_active_out_of_service_count_get(sg
);
1740 standby_out_of_service
= su_standby_out_of_service_count_get(sg
);
1741 if (sg
->saAmfSGNumPrefActiveSUs
> 0) {
1743 active_sus_needed
= div_round (
1744 sg_si_count_get (sg
),
1745 sg
->saAmfSGMaxActiveSIsperSUs
);
1747 log_printf (LOG_LEVEL_ERROR
, "ERROR: saAmfSGNumPrefActiveSUs == 0 !!");
1748 openais_exit_error (AIS_DONE_FATAL_ERR
);
1751 if (sg
->saAmfSGNumPrefStandbySUs
> 0) {
1752 standby_sus_needed
= div_round (
1753 sg_si_count_get (sg
),
1754 sg
->saAmfSGMaxStandbySIsperSUs
);
1756 log_printf (LOG_LEVEL_ERROR
, "ERROR: saAmfSGNumPrefStandbySUs == 0 !!");
1757 openais_exit_error (AIS_DONE_FATAL_ERR
);
1761 dprintf ("(inservice=%d) (active_sus_needed=%d) (standby_sus_needed=%d)"
1763 inservice_count
, active_sus_needed
, standby_sus_needed
);
1765 /* Determine number of active and standby service units
1766 * to assign based upon reduction procedure
1768 if ((inservice_count
< active_sus_needed
- active_out_of_service
)) {
1769 dprintf ("assignment VI - partial assignment with SIs drop outs\n");
1771 su_active_assign
= inservice_count
;
1772 su_standby_assign
= 0;
1773 su_spare_assign
= 0;
1775 if ((inservice_count
< active_sus_needed
- active_out_of_service
+
1776 standby_sus_needed
)) {
1777 dprintf ("assignment V - partial assignment with reduction of"
1778 " standby units\n");
1780 su_active_assign
= active_sus_needed
;
1781 su_standby_assign
= inservice_count
- active_sus_needed
- active_out_of_service
;
1782 su_spare_assign
= 0;
1784 if ((inservice_count
< sg
->saAmfSGNumPrefActiveSUs
+ standby_sus_needed
)) {
1785 dprintf ("IV: full assignment with reduction of active service"
1787 su_active_assign
= inservice_count
- standby_sus_needed
;
1788 su_standby_assign
= standby_sus_needed
;
1789 su_spare_assign
= 0;
1791 if ((inservice_count
<
1792 sg
->saAmfSGNumPrefActiveSUs
+ sg
->saAmfSGNumPrefStandbySUs
)) {
1793 dprintf ("III: full assignment with reduction of standby service"
1795 su_active_assign
= sg
->saAmfSGNumPrefActiveSUs
;
1796 su_standby_assign
= inservice_count
- sg
->saAmfSGNumPrefActiveSUs
;
1797 su_spare_assign
= 0;
1799 if ((inservice_count
==
1800 sg
->saAmfSGNumPrefActiveSUs
+ sg
->saAmfSGNumPrefStandbySUs
)) {
1801 if (sg
->saAmfSGNumPrefInserviceSUs
> inservice_count
) {
1802 dprintf ("II: full assignment with spare reduction\n");
1804 dprintf ("II: full assignment without spares\n");
1807 su_active_assign
= sg
->saAmfSGNumPrefActiveSUs
;
1808 su_standby_assign
= sg
->saAmfSGNumPrefStandbySUs
;
1809 su_spare_assign
= 0;
1811 dprintf ("I: full assignment with spares\n");
1812 su_active_assign
= sg
->saAmfSGNumPrefActiveSUs
;
1813 su_standby_assign
= sg
->saAmfSGNumPrefStandbySUs
;
1814 su_spare_assign
= inservice_count
-
1815 sg
->saAmfSGNumPrefActiveSUs
- sg
->saAmfSGNumPrefStandbySUs
;
1818 dprintf ("(inservice=%d) (assigning active=%d) (assigning standby=%d)"
1819 " (assigning spares=%d)\n",
1820 inservice_count
, su_active_assign
, su_standby_assign
, su_spare_assign
);
1822 if (inservice_count
> 0) {
1823 assigned
= sg_assign_active_nplusm (sg
, su_active_assign
);
1824 assigned
+= sg_assign_standby_nplusm (sg
, su_standby_assign
);
1825 sg
->saAmfSGNumCurrAssignedSUs
= inservice_count
;
1828 * Phase 2: do the actual assignment to the component
1829 * TODO: first do active, then standby
1833 struct amf_si_assignment
*si_assignment
;
1835 for (si
= sg
->application
->si_head
; si
!= NULL
; si
= si
->next
) {
1836 if (name_match (&si
->saAmfSIProtectedbySG
, &sg
->name
)) {
1837 for (si_assignment
= si
->assigned_sis
;
1838 si_assignment
!= NULL
;
1839 si_assignment
= si_assignment
->next
) {
1841 if (si_assignment
->requested_ha_state
!=
1842 si_assignment
->saAmfSISUHAState
) {
1843 amf_si_ha_state_assume (
1844 si_assignment
, assign_si_assumed_cbfn
);
1852 LEAVE ("'%s'", sg
->name
.value
);
1857 static void remove_si_in_scope (amf_sg_t
*sg
, amf_si_t
*si
)
1861 amf_si_t
**sis
= sg
->recovery_scope
.sis
;
1862 amf_si_t
**new_sis
= amf_calloc (1, sizeof (amf_si_t
*));
1864 for (i
= 0,j
= 0; sis
[i
] != NULL
; i
++) {
1868 new_sis
[j
] = sis
[i
];
1869 new_sis
= amf_realloc (new_sis
, j
+ sizeof (amf_si_t
*));
1873 sg
->recovery_scope
.sis
= new_sis
;
1878 static void remove_sis_for_term_failed_su_from_scope (amf_sg_t
*sg
,
1881 amf_comp_t
*component
;
1884 * foreach component with presense state termiantion failed in su
1887 for (component
= su
->comp_head
; component
!= NULL
;
1888 component
= component
->next
) {
1890 amf_csi_assignment_t
*csi_assignment
;
1892 if (component
->saAmfCompPresenceState
!=
1893 SA_AMF_PRESENCE_INSTANTIATION_FAILED
) {
1897 csi_assignment
= amf_comp_get_next_csi_assignment (component
, NULL
);
1899 while (csi_assignment
!= NULL
) {
1901 remove_si_in_scope (sg
, csi_assignment
->csi
->si
);
1902 csi_assignment
= amf_comp_get_next_csi_assignment (component
, NULL
);
1909 * This function returns 1 if the redundancy model is N plus M and at least one
1910 * component of the specified SU has an active HA-state, else the function
1916 static int is_comp_in_active_ha_state_nplusm (
1917 amf_sg_t
*sg
, amf_su_t
*su
)
1919 amf_comp_t
*component
;
1920 amf_csi_assignment_t
*csi_assignment
;
1921 int comp_is_in_active_ha_state
= 0;
1923 if(sg
->saAmfSGRedundancyModel
== SA_AMF_NPM_REDUNDANCY_MODEL
) {
1924 for (component
= su
->comp_head
; component
!= NULL
;
1925 component
= component
->next
) {
1926 csi_assignment
= amf_comp_get_next_csi_assignment(component
, NULL
);
1927 while (csi_assignment
!= NULL
) {
1928 if (csi_assignment
->saAmfCSICompHAState
== SA_AMF_HA_ACTIVE
) {
1929 comp_is_in_active_ha_state
= 1;
1932 csi_assignment
= amf_comp_get_next_csi_assignment(component
,
1938 return comp_is_in_active_ha_state
;
1942 * This function handles a change of presence state reported by an SU contained
1943 * in specified SG. The new presence state is INSTANTIATED.
1947 static void sg_su_state_changed_to_instantiated (struct amf_sg
*sg
, struct amf_su
*su
)
1949 ENTER("%s %s",sg
->name
.value
, su
->name
.value
);
1950 switch (sg
->avail_state
) {
1951 case SG_AC_InstantiatingServiceUnits
:
1952 if (no_su_has_presence_state(sg
, sg
->node_to_start
,
1953 SA_AMF_PRESENCE_INSTANTIATING
)) {
1954 acsm_enter_idle (sg
);
1957 case SG_AC_ReparingSu
:
1958 if (no_su_has_presence_state(sg
, sg
->node_to_start
,
1959 SA_AMF_PRESENCE_INSTANTIATING
)) {
1960 if (all_su_in_scope_has_either_of_three_presence_state(
1962 SA_AMF_PRESENCE_INSTANTIATED
,
1963 SA_AMF_PRESENCE_INSTANTIATION_FAILED
,
1964 SA_AMF_PRESENCE_UNINSTANTIATED
)) {
1965 su
->sg
->avail_state
= SG_AC_AssigningWorkload
;
1966 if (assign_si (sg
, 0) == 0) {
1967 acsm_enter_idle (sg
);
1970 dprintf ("avail-state: %u", sg
->avail_state
);
1976 dprintf ("avail-state: %u", sg
->avail_state
);
1983 * This function handles a change of presence state reported by an SU contained
1984 * in specified SG. The new presence state is UNINSTANTIATED.
1988 static void amf_sg_su_state_changed_to_uninstantiated (amf_sg_t
*sg
,
1991 ENTER("%s %s",sg
->name
.value
, su
->name
.value
);
1992 switch (sg
->avail_state
) {
1993 case SG_AC_TerminatingSuspected
:
1994 if (no_su_has_presence_state(sg
, sg
->node_to_start
,
1995 SA_AMF_PRESENCE_TERMINATING
)) {
1996 if (all_su_in_scope_has_either_two_presence_state (sg
,
1997 SA_AMF_PRESENCE_UNINSTANTIATED
,
1998 SA_AMF_PRESENCE_TERMINATION_FAILED
)) {
2000 delete_si_assignments_in_scope (sg
);
2002 if (is_any_si_in_scope_assigned_standby (sg
)) {
2003 remove_all_suspected_sus (sg
);
2004 acsm_enter_removing_standby_assignments (sg
);
2005 } else { /*is_no_si_in_scope_assigned_standby*/
2006 remove_all_suspected_sus (sg
);
2007 acsm_enter_assigning_standby_to_spare (sg
);
2012 case SG_AC_ReparingSu
:
2013 if (no_su_has_presence_state(sg
, sg
->node_to_start
,
2014 SA_AMF_PRESENCE_TERMINATING
)) {
2015 if (all_su_in_scope_has_either_of_three_presence_state(
2017 SA_AMF_PRESENCE_INSTANTIATED
,
2018 SA_AMF_PRESENCE_INSTANTIATION_FAILED
,
2019 SA_AMF_PRESENCE_UNINSTANTIATED
)) {
2020 su
->sg
->avail_state
= SG_AC_AssigningWorkload
;
2021 if (assign_si (sg
, 0) == 0) {
2022 acsm_enter_idle (sg
);
2028 log_printf (LOG_ERR
, "sg avail_state = %d", sg
->avail_state
);
2035 * This function handles a change of presence state reported by an SU contained
2036 * in specified SG. The new presence state is TERMINATION_FAILED.
2040 static void amf_sg_su_state_changed_to_termination_failed (amf_sg_t
*sg
,
2043 ENTER("%s %s",sg
->name
.value
, su
->name
.value
);
2044 if (no_su_has_presence_state(sg
, sg
->node_to_start
,
2045 SA_AMF_PRESENCE_TERMINATING
)) {
2046 if (is_comp_in_active_ha_state_nplusm (sg
, su
)) {
2047 acsm_enter_idle (sg
);
2051 if (all_su_in_scope_has_either_two_presence_state (sg
,
2052 SA_AMF_PRESENCE_UNINSTANTIATED
,
2053 SA_AMF_PRESENCE_TERMINATION_FAILED
)) {
2055 delete_si_assignments_in_scope (sg
);
2057 if (is_any_si_in_scope_assigned_standby (sg
)) {
2058 remove_all_suspected_sus (sg
);
2059 acsm_enter_removing_standby_assignments (sg
);
2060 } else { /*is_no_si_in_scope_assigned_standby*/
2061 remove_all_suspected_sus (sg
);
2062 acsm_enter_assigning_standby_to_spare (sg
);
2071 * This function handles a change of presence state reported by an SU contained
2072 * in specified SG. The new presence state is INSTANTIATION_FAILED.
2076 static void amf_sg_su_state_changed_to_instantiation_failed (amf_sg_t
*sg
,
2079 ENTER("%s %s",sg
->name
.value
, su
->name
.value
);
2080 switch (sg
->avail_state
) {
2081 case SG_AC_InstantiatingServiceUnits
:
2082 if (no_su_has_presence_state(sg
, sg
->node_to_start
,
2083 SA_AMF_PRESENCE_INSTANTIATING
)) {
2084 acsm_enter_idle (sg
);
2087 case SG_AC_ReparingSu
:
2088 if (no_su_has_presence_state(sg
, sg
->node_to_start
,
2089 SA_AMF_PRESENCE_INSTANTIATING
)) {
2090 if (all_su_in_scope_has_either_of_three_presence_state(
2092 SA_AMF_PRESENCE_INSTANTIATED
,
2093 SA_AMF_PRESENCE_INSTANTIATION_FAILED
,
2094 SA_AMF_PRESENCE_UNINSTANTIATED
)) {
2095 su
->sg
->avail_state
= SG_AC_AssigningWorkload
;
2096 if (assign_si (sg
, 0) == 0) {
2097 acsm_enter_idle (sg
);
2103 /* TODO: Insert the assert (0) until solving defers in SU */
2104 dprintf("sg->avail_state = %d", sg
->avail_state
);
2109 /******************************************************************************
2111 *****************************************************************************/
2114 * This function starts all SUs in the SG or all SUs on a specified node.
2116 * @param node - Node on which all SUs shall be started or
2117 * NULL indicating that all SUs on all nodes shall be started.
2118 * @return - No of SUs that has been attempted to start
2120 int amf_sg_start (struct amf_sg
*sg
, struct amf_node
*node
)
2123 sg
->recovery_scope
.event_type
= SG_START_EV
;
2124 ENTER ("'%s'", sg
->name
.value
);
2125 int instantiated_sus
= 0;
2127 switch (sg
->avail_state
) {
2128 case SG_AC_InstantiatingServiceUnits
:
2132 sg_avail_control_state_t old_avail_state
= sg
->avail_state
;
2134 ENTER ("'%s'", sg
->name
.value
);
2136 sg
->node_to_start
= node
;
2138 sg
->avail_state
= SG_AC_InstantiatingServiceUnits
;
2140 for (su
= sg
->su_head
;
2142 (instantiated_sus
< sg
->saAmfSGNumPrefInserviceSUs
);
2145 if (is_cluster_start (node
)) {
2147 amf_su_instantiate (su
);
2150 } else { /*is_not_cluster_start*/
2153 * Node start, match if SU is hosted on the
2157 if (name_match (&node
->name
,
2158 &su
->saAmfSUHostedByNode
)) {
2159 amf_su_instantiate (su
);
2165 if (instantiated_sus
== 0) {
2166 sg
->avail_state
= old_avail_state
;
2170 case SG_AC_DeactivatingDependantWorkload
:
2171 case SG_AC_TerminatingSuspected
:
2172 case SG_AC_ActivatingStandby
:
2173 case SG_AC_AssigningStandbyToSpare
:
2174 case SG_AC_ReparingComponent
:
2175 case SG_AC_ReparingSu
:
2176 case SG_AC_AssigningOnRequest
:
2177 case SG_AC_RemovingAssignment
:
2178 case SG_AC_AssigningActiveworkload
:
2179 case SG_AC_AssigningAutoAdjust
:
2180 case SG_AC_AssigningWorkload
:
2181 case SG_AC_WaitingAfterOperationFailed
:
2182 case SG_AC_RemovingStandbyAssignments
:
2187 return instantiated_sus
;
2191 * This function initiates assignment of the subset of the workload which
2192 * matches the specified workload dependency level, to all SUs contained in the
2193 * SG according to the requirements specified in the configuration.
2195 * @param dependency_level - Dependency level to assign
2196 * @return - No of SUs that has been attempted to start
2198 int amf_sg_assign_si_req (struct amf_sg
*sg
, int dependency_level
)
2200 /* TODO: Introduce state control in this function
2202 int posible_to_assign_si
;
2203 sg
->recovery_scope
.event_type
= SG_ASSIGN_SI_EV
;
2204 sg
->avail_state
= SG_AC_AssigningOnRequest
;
2206 if ((posible_to_assign_si
= assign_si (sg
, dependency_level
)) == 0) {
2207 acsm_enter_idle (sg
);
2209 return posible_to_assign_si
;
2213 * This function is called because an error has been detected and the analysis
2214 * (done elsewhere) indicated that this error shall be recovered by a Node
2215 * failover. This function initiates the recovery action 'Node failover'.
2217 * @param su - SU to failover
2220 void amf_sg_failover_node_req (struct amf_sg
*sg
, struct amf_node
*node
)
2222 ENTER("'%s, %s'",node
->name
.value
, sg
->name
.value
);
2223 sg_event_t sg_event
;
2225 switch (sg
->avail_state
) {
2227 set_scope_for_failover_node(sg
, node
);
2228 if (has_any_su_in_scope_active_workload (sg
)) {
2229 acsm_enter_deactivating_dependent_workload (sg
);
2231 amf_su_t
**sus
= sg
->recovery_scope
.sus
;
2234 * Select next state depending on if some
2235 * SU in the scope needs to be terminated.
2237 while (*sus
!= NULL
) {
2239 amf_su_t
*su
= *sus
;
2240 ENTER("SU %s pr_state='%d'",su
->name
.value
,
2241 su
->saAmfSUPresenceState
);
2243 if (su_presense_state_is_ored (su
,
2244 SA_AMF_PRESENCE_UNINSTANTIATED
,
2245 SA_AMF_PRESENCE_TERMINATION_FAILED
,
2246 SA_AMF_PRESENCE_INSTANTIATION_FAILED
)) {
2254 acsm_enter_terminating_suspected (sg
);
2256 delete_si_assignments_in_scope (sg
);
2257 acsm_enter_idle (sg
);
2262 case SG_AC_DeactivatingDependantWorkload
:
2263 case SG_AC_TerminatingSuspected
:
2264 case SG_AC_ActivatingStandby
:
2265 case SG_AC_AssigningStandbyToSpare
:
2266 case SG_AC_ReparingComponent
:
2267 case SG_AC_ReparingSu
:
2268 case SG_AC_AssigningOnRequest
:
2269 case SG_AC_InstantiatingServiceUnits
:
2270 case SG_AC_RemovingAssignment
:
2271 case SG_AC_AssigningActiveworkload
:
2272 case SG_AC_AssigningAutoAdjust
:
2273 case SG_AC_AssigningWorkload
:
2274 case SG_AC_WaitingAfterOperationFailed
:
2275 case SG_AC_RemovingStandbyAssignments
:
2276 sg_set_event (SG_FAILOVER_NODE_EV
, sg
, 0, 0, node
, &sg_event
);
2277 sg_defer_event (SG_FAILOVER_NODE_EV
, &sg_event
);
2287 * This function is called because an error has been detected and the analysis
2288 * (done elsewhere) indicated that this error shall be recovered by an SU
2289 * failover. This function initiates the recovery action 'SU failover'.
2291 * @param su - SU to failover
2294 void amf_sg_failover_su_req (struct amf_sg
*sg
, struct amf_su
*su
,
2295 struct amf_node
*node
)
2297 ENTER ("%s", su
->name
.value
);
2298 sg_event_t sg_event
;
2300 switch (sg
->avail_state
) {
2302 su
->su_failover_cnt
+= 1;
2303 set_scope_for_failover_su (sg
, su
);
2304 if (has_any_su_in_scope_active_workload (sg
)) {
2305 acsm_enter_deactivating_dependent_workload (sg
);
2307 acsm_enter_terminating_suspected (sg
);
2310 case SG_AC_DeactivatingDependantWorkload
:
2311 case SG_AC_TerminatingSuspected
:
2312 case SG_AC_ActivatingStandby
:
2313 case SG_AC_AssigningStandbyToSpare
:
2314 case SG_AC_ReparingComponent
:
2315 case SG_AC_ReparingSu
:
2316 case SG_AC_AssigningOnRequest
:
2317 case SG_AC_InstantiatingServiceUnits
:
2318 case SG_AC_RemovingAssignment
:
2319 case SG_AC_AssigningActiveworkload
:
2320 case SG_AC_AssigningAutoAdjust
:
2321 case SG_AC_AssigningWorkload
:
2322 case SG_AC_WaitingAfterOperationFailed
:
2323 case SG_AC_RemovingStandbyAssignments
:
2324 sg_set_event (SG_FAILOVER_SU_EV
, sg
, su
, 0, 0, &sg_event
);
2325 sg_defer_event (SG_FAILOVER_SU_EV
, &sg_event
);
2333 /******************************************************************************
2334 * Event response methods
2335 *****************************************************************************/
2338 void amf_sg_su_state_changed_2 (struct amf_sg
*sg
,
2339 struct amf_su
*su
, SaAmfStateT type
, int state
)
2341 ENTER ("'%s' SU '%s' state %s",
2342 sg
->name
.value
, su
->name
.value
, amf_presence_state(state
));
2344 if (type
== SA_AMF_PRESENCE_STATE
) {
2345 if (state
== SA_AMF_PRESENCE_INSTANTIATED
) {
2346 if (sg
->avail_state
== SG_AC_InstantiatingServiceUnits
) {
2347 if (no_su_has_presence_state(sg
, sg
->node_to_start
,
2348 SA_AMF_PRESENCE_INSTANTIATING
)) {
2349 acsm_enter_idle (sg
);
2351 } else if (sg
->avail_state
== SG_AC_ReparingSu
) {
2352 if (all_su_in_scope_has_either_of_three_presence_state(
2354 SA_AMF_PRESENCE_INSTANTIATED
,
2355 SA_AMF_PRESENCE_INSTANTIATION_FAILED
,
2356 SA_AMF_PRESENCE_UNINSTANTIATED
)) {
2357 su
->sg
->avail_state
= SG_AC_AssigningWorkload
;
2358 if (assign_si (sg
, 0) == 0) {
2359 acsm_enter_idle (sg
);
2363 dprintf ("avail-state: %u", sg
->avail_state
);
2367 dprintf ("avail-state: %u", sg
->avail_state
);
2370 } else if (state
== SA_AMF_PRESENCE_UNINSTANTIATED
) {
2371 if (sg
->avail_state
== SG_AC_TerminatingSuspected
) {
2372 if (all_su_in_scope_has_either_two_presence_state (sg
,
2373 SA_AMF_PRESENCE_UNINSTANTIATED
,
2374 SA_AMF_PRESENCE_TERMINATION_FAILED
)) {
2376 delete_si_assignments_in_scope (sg
);
2378 if (is_any_si_in_scope_assigned_standby (sg
)) {
2379 remove_all_suspected_sus (sg
);
2380 acsm_enter_removing_standby_assignments (sg
);
2381 } else { /*is_no_si_in_scope_assigned_standby*/
2382 remove_all_suspected_sus (sg
);
2383 acsm_enter_assigning_standby_to_spare (sg
);
2386 } else if (sg
->avail_state
== SG_AC_ReparingSu
) {
2387 if (all_su_in_scope_has_either_of_three_presence_state(
2389 SA_AMF_PRESENCE_INSTANTIATED
,
2390 SA_AMF_PRESENCE_INSTANTIATION_FAILED
,
2391 SA_AMF_PRESENCE_UNINSTANTIATED
)) {
2392 su
->sg
->avail_state
= SG_AC_AssigningWorkload
;
2393 if (assign_si (sg
, 0) == 0) {
2394 acsm_enter_idle (sg
);
2397 dprintf("%d",sg
->avail_state
);
2401 } else if (state
== SA_AMF_PRESENCE_TERMINATION_FAILED
) {
2403 if (all_su_in_scope_has_either_two_presence_state (sg
,
2404 SA_AMF_PRESENCE_UNINSTANTIATED
,
2405 SA_AMF_PRESENCE_TERMINATION_FAILED
) &&
2406 is_any_si_in_scope_assigned_standby (sg
)) {
2407 remove_all_suspected_sus (sg
);
2409 acsm_enter_removing_standby_assignments (sg
);
2411 } else if (all_su_in_scope_has_either_two_presence_state (sg
,
2412 SA_AMF_PRESENCE_UNINSTANTIATED
,
2413 SA_AMF_PRESENCE_TERMINATION_FAILED
) &&
2414 !is_any_si_in_scope_assigned_standby (sg
)) {
2416 remove_all_suspected_sus (sg
);
2417 acsm_enter_assigning_standby_to_spare (sg
);
2421 remove_sis_for_term_failed_su_from_scope (sg
, su
);
2423 } else if (state
== SA_AMF_PRESENCE_INSTANTIATING
) {
2425 } else if (state
== SA_AMF_PRESENCE_INSTANTIATION_FAILED
) {
2426 if (sg
->avail_state
== SG_AC_InstantiatingServiceUnits
) {
2427 if (no_su_has_presence_state(sg
, sg
->node_to_start
,
2428 SA_AMF_PRESENCE_INSTANTIATING
)) {
2429 acsm_enter_idle (sg
);
2432 } else if (sg
->avail_state
== SG_AC_ReparingSu
) {
2433 if (all_su_in_scope_has_either_of_three_presence_state(
2435 SA_AMF_PRESENCE_INSTANTIATED
,
2436 SA_AMF_PRESENCE_INSTANTIATION_FAILED
,
2437 SA_AMF_PRESENCE_UNINSTANTIATED
)) {
2438 su
->sg
->avail_state
= SG_AC_AssigningWorkload
;
2439 if (assign_si (sg
, 0) == 0) {
2440 acsm_enter_idle (sg
);
2444 /* TODO: Insert the assert (0) until solving defers in SU */
2445 dprintf("sg->avail_state = %d, su instantiation state = %d",
2446 sg
->avail_state
, state
);
2449 dprintf("sg->avail_state = %d, su instantiation state = %d",
2450 sg
->avail_state
, state
);
2458 * SU indicates to SG that one of its state machines has changed state.
2459 * @param sg - SG which contains the SU that has changed state
2460 * @param su - SU which has changed state
2461 * @param type - Indicates which state machine that has changed state
2462 * @param state - The new state that has been assumed.
2465 void amf_sg_su_state_changed (struct amf_sg
*sg
, struct amf_su
*su
,
2466 SaAmfStateT type
, int state
)
2468 ENTER ("'%s' SU '%s' state %s",
2469 sg
->name
.value
, su
->name
.value
, amf_presence_state(state
));
2471 if (sg
->avail_state
!= SG_AC_Idle
) {
2472 if (type
== SA_AMF_PRESENCE_STATE
) {
2474 case SA_AMF_PRESENCE_INSTANTIATED
:
2475 sg_su_state_changed_to_instantiated(sg
, su
);
2477 case SA_AMF_PRESENCE_UNINSTANTIATED
:
2478 amf_sg_su_state_changed_to_uninstantiated(sg
, su
);
2480 case SA_AMF_PRESENCE_TERMINATION_FAILED
:
2481 amf_sg_su_state_changed_to_termination_failed(sg
, su
);
2483 case SA_AMF_PRESENCE_INSTANTIATING
:
2486 case SA_AMF_PRESENCE_INSTANTIATION_FAILED
:
2487 amf_sg_su_state_changed_to_instantiation_failed(sg
, su
);
2489 case SA_AMF_PRESENCE_TERMINATING
:
2493 dprintf("sg->avail_state = %d, su instantiation state = %d",
2494 sg
->avail_state
, state
);
2503 * Callback function used by SI when there is no dependent SI to
2507 static void dependent_si_deactivated_cbfn2 (struct amf_sg
*sg
)
2509 struct amf_su
**sus
= sg
->recovery_scope
.sus
;
2511 ENTER("'%s'", sg
->name
.value
);
2514 * Select next state depending on if some
2515 * SU in the scope needs to be terminated.
2518 while (*sus
!= NULL
) {
2519 amf_su_t
*su
= *sus
;
2521 ENTER("SU %s pr_state='%d'",su
->name
.value
,
2522 su
->saAmfSUPresenceState
);
2524 if (su_presense_state_is_ored (su
,
2525 SA_AMF_PRESENCE_UNINSTANTIATED
,
2526 SA_AMF_PRESENCE_TERMINATION_FAILED
,
2527 SA_AMF_PRESENCE_INSTANTIATION_FAILED
)) {
2535 acsm_enter_terminating_suspected (sg
);
2537 delete_si_assignments_in_scope(sg
);
2538 acsm_enter_removing_standby_assignments (sg
);
2543 * Callback function used by SI when an SI has been deactivated, i.e.
2544 * transitioned from active HA-state to any other state.
2545 * @param si_assignment
2546 * @param result - Indicates the result of the operation.
2548 static void dependent_si_deactivated_cbfn (
2549 struct amf_si_assignment
*si_assignment
, int result
)
2551 struct amf_sg
*sg
= si_assignment
->su
->sg
;
2552 struct amf_su
**sus
= sg
->recovery_scope
.sus
;
2555 ENTER ("'%s', %d", si_assignment
->si
->name
.value
, result
);
2558 * If all SI assignments for all SUs in the SG are not pending,
2559 * goto next state (TerminatingSuspected).
2561 for (su
= sg
->su_head
; su
!= NULL
; su
= su
->next
) {
2562 struct amf_si_assignment
*si_assignment
;
2563 si_assignment
= amf_su_get_next_si_assignment(su
, NULL
);
2565 while (si_assignment
!= NULL
) {
2566 if (si_assignment
->saAmfSISUHAState
!=
2567 si_assignment
->requested_ha_state
) {
2570 si_assignment
= amf_su_get_next_si_assignment(su
,
2578 sus
= si_assignment
->su
->sg
->recovery_scope
.sus
;
2581 * Select next state depending on if some
2582 * SU in the scope is needs to be terminated.
2585 while (*sus
!= NULL
) {
2586 if (su_presense_state_is_not (*sus
,
2587 SA_AMF_PRESENCE_UNINSTANTIATED
,
2588 SA_AMF_PRESENCE_TERMINATION_FAILED
,
2589 SA_AMF_PRESENCE_INSTANTIATION_FAILED
)) {
2595 acsm_enter_terminating_suspected (sg
);
2597 acsm_enter_removing_standby_assignments (sg
);
2604 * Callback function used by SI to indicate an SI has assumed a new HA-state or
2605 * that the attempt to do so failed.
2606 * @param si_assignment
2607 * @param result - Indicates the result of the operation.
2609 static void assign_si_assumed_cbfn (
2610 struct amf_si_assignment
*si_assignment
, int result
)
2612 struct amf_sg
*sg
= si_assignment
->su
->sg
;
2613 int si_assignment_cnt
= 0;
2614 int confirmed_assignments
= 0;
2616 ENTER ("'%s', %d", si_assignment
->si
->name
.value
, result
);
2619 switch (sg
->avail_state
) {
2620 case SG_AC_AssigningOnRequest
:
2621 if (is_all_si_assigned (sg
)) {
2622 acsm_enter_idle (sg
);
2623 amf_application_sg_assigned (sg
->application
, sg
);
2625 dprintf ("%d, %d", si_assignment_cnt
, confirmed_assignments
);
2628 case SG_AC_AssigningWorkload
:
2630 if (is_all_si_assigned(sg
)) {
2631 acsm_enter_idle (sg
);
2635 case SG_AC_AssigningStandbyToSpare
:
2637 if(is_all_si_assigned (sg
)) {
2639 * All si_assignments has asumed
2640 * Prescense state SA_AMF_HA_STANDBY
2642 switch (sg
->recovery_scope
.event_type
) {
2643 case SG_FAILOVER_NODE_EV
:
2644 acsm_enter_idle (sg
);
2646 case SG_FAILOVER_SU_EV
:
2647 if (sg
->saAmfSGAutoRepair
== SA_TRUE
) {
2648 acsm_enter_repairing_su (sg
);
2656 si_assignment
->saAmfSISUHAState
= SA_AMF_HA_STANDBY
;
2661 dprintf ("%d, %d, %d", sg
->avail_state
, si_assignment_cnt
,
2662 confirmed_assignments
);
2663 amf_runtime_attributes_print (amf_cluster
);
2670 * Callback function used by SI when an SI has been activated, i.e. transitioned
2671 * from any HA-state to an active HA-state.
2672 * @param si_assignment
2673 * @param result - Indicates the result of the operation.
2675 static void standby_su_activated_cbfn (
2676 struct amf_si_assignment
*si_assignment
, int result
)
2678 struct amf_su
**sus
= si_assignment
->su
->sg
->recovery_scope
.sus
;
2679 struct amf_si
**sis
= si_assignment
->su
->sg
->recovery_scope
.sis
;
2681 ENTER ("'%s', %d", si_assignment
->si
->name
.value
, result
);
2684 * If all SI assignments for all SIs in the scope are activated, goto next
2688 while (*sis
!= NULL
) {
2689 if ((*sis
)->assigned_sis
!= NULL
&&
2690 (*sis
)->assigned_sis
->saAmfSISUHAState
!= SA_AMF_HA_ACTIVE
) {
2698 acsm_enter_assigning_standby_to_spare ((*sus
)->sg
);
2702 /******************************************************************************
2704 *****************************************************************************/
2707 * Constructor for SG objects. Adds SG to the list owned by
2708 * the specified application. Always returns a valid SG
2709 * object, out-of-memory problems are handled here. Default
2710 * values are initialized.
2714 * @return struct amf_sg*
2717 struct amf_sg
*amf_sg_new (struct amf_application
*app
, char *name
)
2719 struct amf_sg
*sg
= amf_calloc (1, sizeof (struct amf_sg
));
2721 setSaNameT (&sg
->name
, name
);
2722 sg
->saAmfSGAdminState
= SA_AMF_ADMIN_UNLOCKED
;
2723 sg
->saAmfSGNumPrefActiveSUs
= 1;
2724 sg
->saAmfSGNumPrefStandbySUs
= 1;
2725 sg
->saAmfSGNumPrefInserviceSUs
= ~0;
2726 sg
->saAmfSGNumPrefAssignedSUs
= ~0;
2727 sg
->saAmfSGCompRestartProb
= -1;
2728 sg
->saAmfSGCompRestartMax
= ~0;
2729 sg
->saAmfSGSuRestartProb
= -1;
2730 sg
->saAmfSGSuRestartMax
= ~0;
2731 sg
->saAmfSGAutoAdjustProb
= -1;
2732 sg
->saAmfSGAutoRepair
= SA_TRUE
;
2733 sg
->application
= app
;
2734 sg
->next
= app
->sg_head
;
2736 sg
->deferred_events
= NULL
;
2741 void amf_sg_delete (struct amf_sg
*sg
)
2745 for (su
= sg
->su_head
; su
!= NULL
;) {
2746 struct amf_su
*tmp
= su
;
2748 amf_su_delete (tmp
);
2754 void *amf_sg_serialize (struct amf_sg
*sg
, int *len
)
2757 int offset
= 0, size
= 0;
2759 TRACE8 ("%s", sg
->name
.value
);
2761 buf
= amf_serialize_SaNameT (buf
, &size
, &offset
, &sg
->name
);
2762 buf
= amf_serialize_SaUint32T (buf
, &size
, &offset
, sg
->saAmfSGRedundancyModel
);
2763 buf
= amf_serialize_SaUint32T (
2764 buf
, &size
, &offset
, sg
->saAmfSGAutoAdjust
);
2765 buf
= amf_serialize_SaUint32T (
2766 buf
, &size
, &offset
, sg
->saAmfSGNumPrefActiveSUs
);
2767 buf
= amf_serialize_SaUint32T (
2768 buf
, &size
, &offset
, sg
->saAmfSGNumPrefStandbySUs
);
2769 buf
= amf_serialize_SaUint32T (
2770 buf
, &size
, &offset
, sg
->saAmfSGNumPrefInserviceSUs
);
2771 buf
= amf_serialize_SaUint32T (
2772 buf
, &size
, &offset
, sg
->saAmfSGNumPrefAssignedSUs
);
2773 buf
= amf_serialize_SaUint32T (
2774 buf
, &size
, &offset
, sg
->saAmfSGMaxActiveSIsperSUs
);
2775 buf
= amf_serialize_SaUint32T (
2776 buf
, &size
, &offset
, sg
->saAmfSGMaxStandbySIsperSUs
);
2777 buf
= amf_serialize_SaUint32T (
2778 buf
, &size
, &offset
, sg
->saAmfSGCompRestartProb
);
2779 buf
= amf_serialize_SaUint32T (
2780 buf
, &size
, &offset
, sg
->saAmfSGCompRestartMax
);
2781 buf
= amf_serialize_SaUint32T (
2782 buf
, &size
, &offset
, sg
->saAmfSGSuRestartProb
);
2783 buf
= amf_serialize_SaUint32T (
2784 buf
, &size
, &offset
, sg
->saAmfSGSuRestartMax
);
2785 buf
= amf_serialize_SaUint32T (
2786 buf
, &size
, &offset
, sg
->saAmfSGAutoAdjustProb
);
2787 buf
= amf_serialize_SaUint32T (
2788 buf
, &size
, &offset
, sg
->saAmfSGAutoRepair
);
2789 buf
= amf_serialize_SaUint32T (
2790 buf
, &size
, &offset
, sg
->saAmfSGAdminState
);
2791 buf
= amf_serialize_SaUint32T (
2792 buf
, &size
, &offset
, sg
->saAmfSGNumCurrAssignedSUs
);
2793 buf
= amf_serialize_SaUint32T (
2794 buf
, &size
, &offset
, sg
->saAmfSGNumCurrNonInstantiatedSpareSUs
);
2795 buf
= amf_serialize_SaUint32T (
2796 buf
, &size
, &offset
, sg
->saAmfSGNumCurrInstantiatedSpareSUs
);
2797 buf
= amf_serialize_SaStringT (
2798 buf
, &size
, &offset
, sg
->clccli_path
);
2799 buf
= amf_serialize_SaUint32T (
2800 buf
, &size
, &offset
, sg
->avail_state
);
2801 buf
= amf_serialize_SaUint32T (
2802 buf
, &size
, &offset
, sg
->recovery_scope
.event_type
);
2809 struct amf_sg
*amf_sg_deserialize (struct amf_application
*app
, char *buf
)
2812 struct amf_sg
*sg
= amf_sg_new (app
, "");
2814 tmp
= amf_deserialize_SaNameT (tmp
, &sg
->name
);
2815 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGRedundancyModel
);
2816 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGAutoAdjust
);
2817 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGNumPrefActiveSUs
);
2818 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGNumPrefStandbySUs
);
2819 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGNumPrefInserviceSUs
);
2820 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGNumPrefAssignedSUs
);
2821 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGMaxActiveSIsperSUs
);
2822 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGMaxStandbySIsperSUs
);
2823 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGCompRestartProb
);
2824 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGCompRestartMax
);
2825 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGSuRestartProb
);
2826 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGSuRestartMax
);
2827 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGAutoAdjustProb
);
2828 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGAutoRepair
);
2829 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGAdminState
);
2830 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGNumCurrAssignedSUs
);
2831 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGNumCurrNonInstantiatedSpareSUs
);
2832 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->saAmfSGNumCurrInstantiatedSpareSUs
);
2833 tmp
= amf_deserialize_SaStringT (tmp
, &sg
->clccli_path
);
2834 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->avail_state
);
2835 tmp
= amf_deserialize_SaUint32T (tmp
, &sg
->recovery_scope
.event_type
);
2840 struct amf_sg
*amf_sg_find (struct amf_application
*app
, char *name
)
2844 for (sg
= app
->sg_head
; sg
!= NULL
; sg
= sg
->next
) {
2845 if (sg
->name
.length
== strlen(name
) &&
2846 strncmp (name
, (char*)sg
->name
.value
, sg
->name
.length
) == 0) {