ctdb/config/events.d/00.ctdb

   1 #!/bin/sh
   2
   3 # Event script for ctdb-specific setup and other things that don't fit
   4 # elsewhere.
   5
   6 [ -n "$CTDB_BASE" ] || \
   7     export CTDB_BASE=$(cd -P $(dirname "$0") ; dirname "$PWD")
   8
   9 . $CTDB_BASE/functions
  10 loadconfig
  11
  12 ctdb_setup_service_state_dir "ctdb"
  13
  14 ############################################################
  15
  16 select_tdb_checker ()
  17 {
  18     # Find the best TDB consistency check available.
  19     use_tdb_tool_check=false
  20     if which tdbtool >/dev/null 2>&1 && \
  21         echo "help" | tdbtool | grep -q check ; then
  22
  23         use_tdb_tool_check=true
  24     elif which tdbtool >/dev/null 2>&1 && which tdbdump >/dev/null 2>&1 ; then
  25             cat <<EOF
  26 WARNING: The installed 'tdbtool' does not offer the 'check' subcommand.
  27  Using 'tdbdump' for database checks.
  28  Consider updating 'tdbtool' for better checks!
  29 EOF
  30     elif which tdbdump >/dev/null 2>&1 ; then
  31         cat <<EOF
  32 WARNING: 'tdbtool' is not available.
  33  Using 'tdbdump' to check the databases.
  34  Consider installing a recent 'tdbtool' for better checks!
  35 EOF
  36     else
  37         cat <<EOF
  38 WARNING: Cannot check databases since neither
  39  'tdbdump' nor 'tdbtool check' is available.
  40  Consider installing tdbtool or at least tdbdump!
  41 EOF
  42         return 1
  43     fi
  44 }
  45
  46 check_tdb ()
  47 {
  48     _db="$1"
  49
  50     if $use_tdb_tool_check ; then
  51         # tdbtool always exits with 0  :-(
  52         if tdbtool "$_db" check 2>/dev/null |
  53             grep -q "Database integrity is OK" ; then
  54             return 0
  55         else
  56             return 1
  57         fi
  58     else
  59         tdbdump "$_db" >/dev/null 2>/dev/null
  60         return $?
  61     fi
  62 }
  63
  64 check_persistent_databases ()
  65 {
  66     _dir="${CTDB_DBDIR_PERSISTENT:-${CTDB_DBDIR:-${CTDB_VARDIR}}/persistent}"
  67     mkdir -p "$_dir" 2>/dev/null
  68
  69     [ "${CTDB_MAX_PERSISTENT_CHECK_ERRORS:-0}" = "0" ] || return 0
  70
  71     for _db in $(ls "$_dir/"*.tdb.*[0-9] 2>/dev/null) ; do
  72         check_tdb $_db || {
  73             echo "Persistent database $_db is corrupted! CTDB will not start."
  74             return 1
  75         }
  76     done
  77 }
  78
  79 check_non_persistent_databases ()
  80 {
  81     _dir="${CTDB_DBDIR:-${CTDB_VARDIR}}"
  82     mkdir -p "$_dir" 2>/dev/null
  83
  84     for _db in $(ls "${_dir}/"*.tdb.*[0-9] 2>/dev/null) ; do
  85         check_tdb $_db || {
  86             _backup="${_db}.$(date +'%Y%m%d.%H%M%S.%N').corrupt"
  87             cat <<EOF
  88 WARNING: database ${_db} is corrupted.
  89  Moving to backup ${_backup} for later analysis.
  90 EOF
  91             mv "$_db" "$_backup"
  92
  93             # Now remove excess backups
  94             ls -td "${_db}."*".corrupt" |
  95             tail -n +$((${CTDB_MAX_CORRUPT_DB_BACKUPS:-10} + 1)) |
  96             xargs rm -f
  97
  98         }
  99     done
 100 }
 101
 102 update_config_from_tdb() {
 103
 104     # Pull optional ctdb configuration data out of config.tdb
 105     _key="public_addresses:node#$(ctdb -t 1 xpnn|sed -e 's/.*://')"
 106     _t="$service_state_dir/public_addresses"
 107     rm -f "$_t"
 108
 109     if ctdb pfetch config.tdb "$_key" "$_t" 2>/dev/null && \
 110         [ -s "$_t" -a -n "$CTDB_PUBLIC_ADDRESSES"] && \
 111         ! cmp -s "$_t" "$CTDB_PUBLIC_ADDRESSES" ; then
 112
 113         echo "CTDB public address configuration has changed."
 114         echo "Extracting new configuration from database."
 115         diff "$_t" "$CTDB_PUBLIC_ADDRESSES"
 116         cp "$_t" "$CTDB_PUBLIC_ADDRESSES"
 117         echo "Restarting CTDB"
 118         service ctdb restart &
 119     fi
 120 }
 121
 122 set_ctdb_variables () {
 123     # set any tunables from the config file
 124     set | grep ^CTDB_SET_ | cut -d_ -f3- |
 125     while read v; do
 126         varname=`echo $v | cut -d= -f1`
 127         value=`echo $v | cut -d= -f2`
 128         ctdb setvar $varname $value || return 1
 129         echo "Set $varname to $value"
 130     done
 131 }
 132
 133 monitor_system_memory ()
 134 {
 135     # If monitoring free memory then calculate how much there is
 136     if [ -n "$CTDB_MONITOR_FREE_MEMORY_WARN" -o \
 137         -n "$CTDB_MONITOR_FREE_MEMORY" ] ; then
 138         free_mem=$(free -m | awk '$2 == "buffers/cache:" { print $4 }')
 139     fi
 140
 141     # Shutdown CTDB when memory is below the configured limit
 142     if [ -n "$CTDB_MONITOR_FREE_MEMORY" ] ; then
 143         if [ $free_mem -le $CTDB_MONITOR_FREE_MEMORY ] ; then
 144             echo "CRITICAL: OOM - ${free_mem}MB free <= ${CTDB_MONITOR_FREE_MEMORY}MB (CTDB threshold)"
 145             echo "CRITICAL: Shutting down CTDB!!!"
 146             get_proc "meminfo"
 147             ps auxfww
 148             set_proc "sysrq-trigger" "m"
 149             ctdb disable
 150             sleep 3
 151             ctdb shutdown
 152         fi
 153     fi
 154
 155     # Warn when low on memory
 156     if [ -n "$CTDB_MONITOR_FREE_MEMORY_WARN" ] ; then
 157         if [ $free_mem -le $CTDB_MONITOR_FREE_MEMORY_WARN ] ; then
 158             echo "WARNING: free memory is low - ${free_mem}MB free <=  ${CTDB_MONITOR_FREE_MEMORY_WARN}MB (CTDB threshold)"
 159         fi
 160     fi
 161
 162     # We should never enter swap, so SwapTotal == SwapFree.
 163     if [ "$CTDB_CHECK_SWAP_IS_NOT_USED" = "yes" ] ; then
 164         set -- $(get_proc "meminfo" | awk '$1 ~ /Swap(Total|Free):/ { print $2 }')
 165         if [ "$1" != "$2" ] ; then
 166             echo We are swapping:
 167             get_proc "meminfo"
 168             ps auxfww
 169         fi
 170     fi
 171 }
 172
 173 ############################################################
 174
 175 ctdb_check_args "$@"
 176
 177 case "$1" in
 178      init)
 179         # make sure we have a blank state directory for the scripts to work with
 180         rm -rf $CTDB_VARDIR/state
 181         # Look at the pattern - this should not be -rf!!!
 182         rm -f $ctdb_managed_dir/*
 183         mkdir -p $CTDB_VARDIR/state || {
 184             ret=$?
 185             echo "mkdir -p $CTDB_VARDIR/state - failed - $ret"
 186             exit $ret
 187         }
 188
 189         # make sure we drop any ips that might still be held if
 190         # previous instance of ctdb got killed with -9 or similar
 191         drop_all_public_ips
 192
 193         if select_tdb_checker ; then
 194             check_persistent_databases || exit $?
 195             check_non_persistent_databases
 196         fi
 197         ;;
 198
 199     setup)
 200         # Set any tunables from the config file
 201         set_ctdb_variables || die "Failed to set CTDB tunables"
 202         ;;
 203
 204     startup)
 205         ctdb attach ctdb.tdb persistent
 206         update_config_from_tdb &
 207         ;;
 208     monitor)
 209         monitor_system_memory
 210         ;;
 211
 212     *)
 213         ctdb_standard_event_handler "$@"
 214         ;;
 215 esac
 216
 217 # all OK
 218 exit 0