2 monitoring links to all other nodes to detect dead nodes
5 Copyright (C) Ronnie Sahlberg 2007
7 This program is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3 of the License, or
10 (at your option) any later version.
12 This program is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
17 You should have received a copy of the GNU General Public License
18 along with this program; if not, see <http://www.gnu.org/licenses/>.
22 #include "system/filesys.h"
23 #include "system/network.h"
24 #include "system/wait.h"
29 #include "lib/util/debug.h"
30 #include "lib/util/samba_util.h"
31 #include "lib/util/util_process.h"
33 #include "ctdb_private.h"
35 #include "common/system.h"
36 #include "common/common.h"
37 #include "common/logging.h"
39 struct ctdb_monitor_state
{
40 uint32_t monitoring_mode
;
41 TALLOC_CTX
*monitor_context
;
42 uint32_t next_interval
;
43 uint32_t event_script_timeouts
;
46 static void ctdb_check_health(struct tevent_context
*ev
,
47 struct tevent_timer
*te
,
48 struct timeval t
, void *private_data
);
51 setup the notification script
53 int ctdb_set_notification_script(struct ctdb_context
*ctdb
, const char *script
)
55 ctdb
->notification_script
= talloc_strdup(ctdb
, script
);
56 CTDB_NO_MEMORY(ctdb
, ctdb
->notification_script
);
60 static int ctdb_run_notification_script_child(struct ctdb_context
*ctdb
, const char *event
)
66 if (stat(ctdb
->notification_script
, &st
) != 0) {
67 DEBUG(DEBUG_ERR
,("Could not stat notification script %s. Can not send notifications.\n", ctdb
->notification_script
));
70 if (!(st
.st_mode
& S_IXUSR
)) {
71 DEBUG(DEBUG_ERR
,("Notification script %s is not executable.\n", ctdb
->notification_script
));
75 cmd
= talloc_asprintf(ctdb
, "%s %s\n", ctdb
->notification_script
, event
);
76 CTDB_NO_MEMORY(ctdb
, cmd
);
79 /* if the system() call was successful, translate ret into the
80 return code from the command
83 ret
= WEXITSTATUS(ret
);
86 DEBUG(DEBUG_ERR
,("Notification script \"%s\" failed with error %d\n", cmd
, ret
));
92 void ctdb_run_notification_script(struct ctdb_context
*ctdb
, const char *event
)
96 if (ctdb
->notification_script
== NULL
) {
100 child
= ctdb_fork(ctdb
);
101 if (child
== (pid_t
)-1) {
102 DEBUG(DEBUG_ERR
,("Failed to fork() a notification child process\n"));
108 prctl_set_comment("ctdb_notification");
109 ret
= ctdb_run_notification_script_child(ctdb
, event
);
111 DEBUG(DEBUG_ERR
,(__location__
" Notification script failed\n"));
120 called when a health monitoring event script finishes
122 static void ctdb_health_callback(struct ctdb_context
*ctdb
, int status
, void *p
)
124 struct ctdb_node
*node
= ctdb
->nodes
[ctdb
->pnn
];
126 struct ctdb_node_flag_change c
;
127 uint32_t next_interval
;
130 struct ctdb_srvid_message rd
;
131 const char *state_str
= NULL
;
134 c
.old_flags
= node
->flags
;
140 rddata
.dptr
= (uint8_t *)&rd
;
141 rddata
.dsize
= sizeof(rd
);
143 if (status
== -ECANCELED
) {
144 DEBUG(DEBUG_ERR
,("Monitoring event was cancelled\n"));
145 goto after_change_status
;
148 if (status
== -ETIME
) {
149 ctdb
->monitor
->event_script_timeouts
++;
151 if (ctdb
->monitor
->event_script_timeouts
>=
152 ctdb
->tunable
.monitor_timeout_count
) {
154 ("Maximum monitor timeout count %u reached."
155 " Making node unhealthy\n",
156 ctdb
->tunable
.monitor_timeout_count
));
158 /* We pretend this is OK. */
159 goto after_change_status
;
162 ctdb
->monitor
->event_script_timeouts
= 0;
165 if (status
!= 0 && !(node
->flags
& NODE_FLAGS_UNHEALTHY
)) {
166 DEBUG(DEBUG_NOTICE
,("monitor event failed - disabling node\n"));
167 node
->flags
|= NODE_FLAGS_UNHEALTHY
;
168 ctdb
->monitor
->next_interval
= 5;
170 ctdb_run_notification_script(ctdb
, "unhealthy");
171 } else if (status
== 0 && (node
->flags
& NODE_FLAGS_UNHEALTHY
)) {
172 DEBUG(DEBUG_NOTICE
,("monitor event OK - node re-enabled\n"));
173 node
->flags
&= ~NODE_FLAGS_UNHEALTHY
;
174 ctdb
->monitor
->next_interval
= 5;
176 ctdb_run_notification_script(ctdb
, "healthy");
180 next_interval
= ctdb
->monitor
->next_interval
;
182 ctdb
->monitor
->next_interval
*= 2;
183 if (ctdb
->monitor
->next_interval
> ctdb
->tunable
.monitor_interval
) {
184 ctdb
->monitor
->next_interval
= ctdb
->tunable
.monitor_interval
;
187 tevent_add_timer(ctdb
->ev
, ctdb
->monitor
->monitor_context
,
188 timeval_current_ofs(next_interval
, 0),
189 ctdb_check_health
, ctdb
);
191 if (c
.old_flags
== node
->flags
) {
195 c
.new_flags
= node
->flags
;
197 data
.dptr
= (uint8_t *)&c
;
198 data
.dsize
= sizeof(c
);
200 /* ask the recovery daemon to push these changes out to all nodes */
201 ctdb_daemon_send_message(ctdb
, ctdb
->pnn
,
202 CTDB_SRVID_PUSH_NODE_FLAGS
, data
);
204 if (c
.new_flags
& NODE_FLAGS_UNHEALTHY
) {
205 state_str
= "UNHEALTHY";
207 state_str
= "HEALTHY";
210 /* ask the recmaster to reallocate all addresses */
212 ("Node became %s. Ask recovery master to reallocate IPs\n",
214 ret
= ctdb_daemon_send_message(ctdb
, CTDB_BROADCAST_CONNECTED
, CTDB_SRVID_TAKEOVER_RUN
, rddata
);
218 " Failed to send IP takeover run request\n"));
223 static void ctdb_run_startup(struct tevent_context
*ev
,
224 struct tevent_timer
*te
,
225 struct timeval t
, void *private_data
);
227 called when the startup event script finishes
229 static void ctdb_startup_callback(struct ctdb_context
*ctdb
, int status
, void *p
)
232 DEBUG(DEBUG_ERR
,("startup event failed\n"));
233 tevent_add_timer(ctdb
->ev
, ctdb
->monitor
->monitor_context
,
234 timeval_current_ofs(5, 0),
235 ctdb_run_startup
, ctdb
);
239 DEBUG(DEBUG_NOTICE
,("startup event OK - enabling monitoring\n"));
240 ctdb_set_runstate(ctdb
, CTDB_RUNSTATE_RUNNING
);
241 ctdb
->monitor
->next_interval
= 2;
242 ctdb_run_notification_script(ctdb
, "startup");
244 ctdb
->monitor
->monitoring_mode
= CTDB_MONITORING_ENABLED
;
246 tevent_add_timer(ctdb
->ev
, ctdb
->monitor
->monitor_context
,
247 timeval_current_ofs(ctdb
->monitor
->next_interval
, 0),
248 ctdb_check_health
, ctdb
);
251 static void ctdb_run_startup(struct tevent_context
*ev
,
252 struct tevent_timer
*te
,
253 struct timeval t
, void *private_data
)
255 struct ctdb_context
*ctdb
= talloc_get_type(private_data
,
256 struct ctdb_context
);
259 /* This is necessary to avoid the "startup" event colliding
260 * with the "ipreallocated" event from the takeover run
261 * following the first recovery. We might as well serialise
262 * these things if we can.
264 if (ctdb
->runstate
< CTDB_RUNSTATE_STARTUP
) {
266 ("Not yet in startup runstate. Wait one more second\n"));
267 tevent_add_timer(ctdb
->ev
, ctdb
->monitor
->monitor_context
,
268 timeval_current_ofs(1, 0),
269 ctdb_run_startup
, ctdb
);
273 DEBUG(DEBUG_NOTICE
,("Running the \"startup\" event.\n"));
274 ret
= ctdb_event_script_callback(ctdb
,
275 ctdb
->monitor
->monitor_context
,
276 ctdb_startup_callback
,
277 ctdb
, CTDB_EVENT_STARTUP
, "%s", "");
280 DEBUG(DEBUG_ERR
,("Unable to launch startup event script\n"));
281 tevent_add_timer(ctdb
->ev
, ctdb
->monitor
->monitor_context
,
282 timeval_current_ofs(5, 0),
283 ctdb_run_startup
, ctdb
);
288 wait until we have finished initial recoveries before we start the
291 static void ctdb_wait_until_recovered(struct tevent_context
*ev
,
292 struct tevent_timer
*te
,
293 struct timeval t
, void *private_data
)
295 struct ctdb_context
*ctdb
= talloc_get_type(private_data
, struct ctdb_context
);
297 static int count
= 0;
301 if (count
< 60 || count
%600 == 0) {
302 DEBUG(DEBUG_NOTICE
,("CTDB_WAIT_UNTIL_RECOVERED\n"));
303 if (ctdb
->nodes
[ctdb
->pnn
]->flags
& NODE_FLAGS_STOPPED
) {
304 DEBUG(DEBUG_NOTICE
,("Node is STOPPED. Node will NOT recover.\n"));
308 if (ctdb
->vnn_map
->generation
== INVALID_GENERATION
) {
309 ctdb
->db_persistent_startup_generation
= INVALID_GENERATION
;
311 tevent_add_timer(ctdb
->ev
, ctdb
->monitor
->monitor_context
,
312 timeval_current_ofs(1, 0),
313 ctdb_wait_until_recovered
, ctdb
);
317 if (ctdb
->recovery_mode
!= CTDB_RECOVERY_NORMAL
) {
318 ctdb
->db_persistent_startup_generation
= INVALID_GENERATION
;
320 DEBUG(DEBUG_NOTICE
,(__location__
" in recovery. Wait one more second\n"));
321 tevent_add_timer(ctdb
->ev
, ctdb
->monitor
->monitor_context
,
322 timeval_current_ofs(1, 0),
323 ctdb_wait_until_recovered
, ctdb
);
328 if (!fast_start
&& timeval_elapsed(&ctdb
->last_recovery_finished
) < (ctdb
->tunable
.rerecovery_timeout
+ 3)) {
329 ctdb
->db_persistent_startup_generation
= INVALID_GENERATION
;
331 DEBUG(DEBUG_NOTICE
,(__location__
" wait for pending recoveries to end. Wait one more second.\n"));
333 tevent_add_timer(ctdb
->ev
, ctdb
->monitor
->monitor_context
,
334 timeval_current_ofs(1, 0),
335 ctdb_wait_until_recovered
, ctdb
);
339 if (ctdb
->vnn_map
->generation
== ctdb
->db_persistent_startup_generation
) {
340 DEBUG(DEBUG_INFO
,(__location__
" skip ctdb_recheck_persistent_health() "
341 "until the next recovery\n"));
342 tevent_add_timer(ctdb
->ev
, ctdb
->monitor
->monitor_context
,
343 timeval_current_ofs(1, 0),
344 ctdb_wait_until_recovered
, ctdb
);
348 ctdb
->db_persistent_startup_generation
= ctdb
->vnn_map
->generation
;
349 ret
= ctdb_recheck_persistent_health(ctdb
);
351 ctdb
->db_persistent_check_errors
++;
352 if (ctdb
->db_persistent_check_errors
< ctdb
->max_persistent_check_errors
) {
354 (__location__
"ctdb_recheck_persistent_health() "
355 "failed (%llu of %llu times) - retry later\n",
356 (unsigned long long)ctdb
->db_persistent_check_errors
,
357 (unsigned long long)ctdb
->max_persistent_check_errors
));
358 tevent_add_timer(ctdb
->ev
,
359 ctdb
->monitor
->monitor_context
,
360 timeval_current_ofs(1, 0),
361 ctdb_wait_until_recovered
, ctdb
);
364 DEBUG(DEBUG_ALERT
,(__location__
365 "ctdb_recheck_persistent_health() failed (%llu times) - prepare shutdown\n",
366 (unsigned long long)ctdb
->db_persistent_check_errors
));
367 ctdb_shutdown_sequence(ctdb
, 11);
368 /* In case above returns due to duplicate shutdown */
371 ctdb
->db_persistent_check_errors
= 0;
373 tevent_add_timer(ctdb
->ev
, ctdb
->monitor
->monitor_context
,
374 timeval_current(), ctdb_run_startup
, ctdb
);
379 see if the event scripts think we are healthy
381 static void ctdb_check_health(struct tevent_context
*ev
,
382 struct tevent_timer
*te
,
383 struct timeval t
, void *private_data
)
385 struct ctdb_context
*ctdb
= talloc_get_type(private_data
, struct ctdb_context
);
386 bool skip_monitoring
= false;
389 if (ctdb
->recovery_mode
!= CTDB_RECOVERY_NORMAL
||
390 ctdb
->monitor
->monitoring_mode
== CTDB_MONITORING_DISABLED
) {
391 skip_monitoring
= true;
393 if (ctdb_db_all_frozen(ctdb
)) {
395 ("Skip monitoring since databases are frozen\n"));
396 skip_monitoring
= true;
400 if (skip_monitoring
) {
401 tevent_add_timer(ctdb
->ev
, ctdb
->monitor
->monitor_context
,
402 timeval_current_ofs(ctdb
->monitor
->next_interval
, 0),
403 ctdb_check_health
, ctdb
);
407 ret
= ctdb_event_script_callback(ctdb
,
408 ctdb
->monitor
->monitor_context
,
409 ctdb_health_callback
,
410 ctdb
, CTDB_EVENT_MONITOR
, "%s", "");
412 DEBUG(DEBUG_ERR
,("Unable to launch monitor event script\n"));
413 ctdb
->monitor
->next_interval
= 5;
414 tevent_add_timer(ctdb
->ev
, ctdb
->monitor
->monitor_context
,
415 timeval_current_ofs(5, 0),
416 ctdb_check_health
, ctdb
);
421 (Temporaily) Disabling monitoring will stop the monitor event scripts
422 from running but node health checks will still occur
424 void ctdb_disable_monitoring(struct ctdb_context
*ctdb
)
426 ctdb
->monitor
->monitoring_mode
= CTDB_MONITORING_DISABLED
;
427 DEBUG(DEBUG_INFO
,("Monitoring has been disabled\n"));
431 Re-enable running monitor events after they have been disabled
433 void ctdb_enable_monitoring(struct ctdb_context
*ctdb
)
435 ctdb
->monitor
->monitoring_mode
= CTDB_MONITORING_ENABLED
;
436 ctdb
->monitor
->next_interval
= 5;
437 DEBUG(DEBUG_INFO
,("Monitoring has been enabled\n"));
440 /* stop any monitoring
441 this should only be done when shutting down the daemon
443 void ctdb_stop_monitoring(struct ctdb_context
*ctdb
)
445 talloc_free(ctdb
->monitor
->monitor_context
);
446 ctdb
->monitor
->monitor_context
= NULL
;
448 ctdb
->monitor
->monitoring_mode
= CTDB_MONITORING_DISABLED
;
449 ctdb
->monitor
->next_interval
= 5;
450 DEBUG(DEBUG_NOTICE
,("Monitoring has been stopped\n"));
454 start watching for nodes that might be dead
456 void ctdb_wait_for_first_recovery(struct ctdb_context
*ctdb
)
458 ctdb_set_runstate(ctdb
, CTDB_RUNSTATE_FIRST_RECOVERY
);
460 ctdb
->monitor
= talloc(ctdb
, struct ctdb_monitor_state
);
461 CTDB_NO_MEMORY_FATAL(ctdb
, ctdb
->monitor
);
463 ctdb
->monitor
->monitor_context
= talloc_new(ctdb
->monitor
);
464 CTDB_NO_MEMORY_FATAL(ctdb
, ctdb
->monitor
->monitor_context
);
466 tevent_add_timer(ctdb
->ev
, ctdb
->monitor
->monitor_context
,
467 timeval_current_ofs(1, 0),
468 ctdb_wait_until_recovered
, ctdb
);
473 modify flags on a node
475 int32_t ctdb_control_modflags(struct ctdb_context
*ctdb
, TDB_DATA indata
)
477 struct ctdb_node_flag_change
*c
= (struct ctdb_node_flag_change
*)indata
.dptr
;
478 struct ctdb_node
*node
;
481 if (c
->pnn
>= ctdb
->num_nodes
) {
482 DEBUG(DEBUG_ERR
,(__location__
" Node %d is invalid, num_nodes :%d\n", c
->pnn
, ctdb
->num_nodes
));
486 node
= ctdb
->nodes
[c
->pnn
];
487 old_flags
= node
->flags
;
488 if (c
->pnn
!= ctdb
->pnn
) {
489 c
->old_flags
= node
->flags
;
491 node
->flags
= c
->new_flags
& ~NODE_FLAGS_DISCONNECTED
;
492 node
->flags
|= (c
->old_flags
& NODE_FLAGS_DISCONNECTED
);
494 /* we don't let other nodes modify our STOPPED status */
495 if (c
->pnn
== ctdb
->pnn
) {
496 node
->flags
&= ~NODE_FLAGS_STOPPED
;
497 if (old_flags
& NODE_FLAGS_STOPPED
) {
498 node
->flags
|= NODE_FLAGS_STOPPED
;
502 /* we don't let other nodes modify our BANNED status */
503 if (c
->pnn
== ctdb
->pnn
) {
504 node
->flags
&= ~NODE_FLAGS_BANNED
;
505 if (old_flags
& NODE_FLAGS_BANNED
) {
506 node
->flags
|= NODE_FLAGS_BANNED
;
510 if (node
->flags
== c
->old_flags
) {
511 DEBUG(DEBUG_INFO
, ("Control modflags on node %u - Unchanged - flags 0x%x\n", c
->pnn
, node
->flags
));
515 DEBUG(DEBUG_INFO
, ("Control modflags on node %u - flags now 0x%x\n", c
->pnn
, node
->flags
));
517 if (node
->flags
== 0 && ctdb
->runstate
<= CTDB_RUNSTATE_STARTUP
) {
518 DEBUG(DEBUG_ERR
, (__location__
" Node %u became healthy - force recovery for startup\n",
520 ctdb
->recovery_mode
= CTDB_RECOVERY_ACTIVE
;
523 /* tell the recovery daemon something has changed */
524 c
->new_flags
= node
->flags
;
525 ctdb_daemon_send_message(ctdb
, ctdb
->pnn
,
526 CTDB_SRVID_SET_NODE_FLAGS
, indata
);
532 return the monitoring mode
534 int32_t ctdb_monitoring_mode(struct ctdb_context
*ctdb
)
536 if (ctdb
->monitor
== NULL
) {
537 return CTDB_MONITORING_DISABLED
;
539 return ctdb
->monitor
->monitoring_mode
;
543 * Check if monitoring has been stopped
545 bool ctdb_stopped_monitoring(struct ctdb_context
*ctdb
)
547 return (ctdb
->monitor
->monitor_context
== NULL
? true : false);