ctdb/config/events.d/05.system

   1 #!/bin/sh
   2 # ctdb event script for checking local file system utilization
   3
   4 [ -n "$CTDB_BASE" ] || \
   5     export CTDB_BASE=$(cd -P $(dirname "$0") ; dirname "$PWD")
   6
   7 . $CTDB_BASE/functions
   8 loadconfig
   9
  10 validate_percentage ()
  11 {
  12     case "$1" in
  13         "") return 1 ;;  # A failure that doesn't need a warning
  14         [0-9]|[0-9][0-9]|100) return 0 ;;
  15         *) echo "WARNING: ${1} is an invalid percentage${2:+ in \"}${2}${2:+\"} check"
  16            return 1
  17     esac
  18 }
  19
  20 monitor_filesystem_usage ()
  21 {
  22     # Check each specified filesystem, specified in format
  23     # <fs_mount>:<fs_warn_threshold>[:fs_unhealthy_threshold]
  24     for _fs in $CTDB_MONITOR_FILESYSTEM_USAGE ; do
  25         _fs_mount="${_fs%%:*}"
  26         _fs_thresholds="${_fs#*:}"
  27
  28         if [ ! -d "$_fs_mount" ]; then
  29             echo "WARNING: Directory ${_fs_mount} does not exist"
  30             continue
  31         fi
  32
  33         # Get current utilization
  34         _fs_usage=$(df -kP "$_fs_mount" | \
  35                            sed -n -e 's@.*[[:space:]]\([[:digit:]]*\)%.*@\1@p')
  36         if [ -z "$_fs_usage" ] ; then
  37             echo "WARNING: Unable to get FS utilization for ${_fs_mount}"
  38             continue
  39         fi
  40
  41         case "$_fs_thresholds" in
  42             *:*)
  43                 _fs_warn_threshold="${_fs_thresholds%:*}"
  44                 _fs_unhealthy_threshold="${_fs_thresholds#*:}"
  45                 ;;
  46             *)
  47                 _fs_warn_threshold="$_fs_thresholds"
  48                 _fs_unhealthy_threshold=""
  49         esac
  50
  51         if validate_percentage "$_fs_unhealthy_threshold" "$_fs" ; then
  52             if [ "$_fs_usage" -ge "$_fs_unhealthy_threshold" ] ; then
  53                 die "ERROR: Filesystem ${_fs_mount} utilization ${_fs_usage}% >= threshold ${_fs_unhealthy_threshold}%"
  54             fi
  55         fi
  56
  57         if validate_percentage "$_fs_warn_threshold" "$_fs" ; then
  58             if [ "$_fs_usage" -ge "$_fs_warn_threshold" ] ; then
  59                 echo "WARNING: Filesystem ${_fs_mount} utilization ${_fs_usage}% >= threshold ${_fs_warn_threshold}%"
  60             fi
  61         fi
  62     done
  63 }
  64
  65 monitor_memory_usage ()
  66 {
  67     if [ -z "$CTDB_MONITOR_FREE_MEMORY_WARN" -a \
  68          -z "$CTDB_MONITOR_FREE_MEMORY" -a \
  69          "$CTDB_CHECK_SWAP_IS_NOT_USED" != "yes" ] ; then
  70         return
  71     fi
  72
  73     _meminfo=$(get_proc "meminfo")
  74     set -- $(echo "$_meminfo" | awk '
  75 $1 == "MemAvailable:" { memavail += $2 }
  76 $1 == "MemFree:"      { memfree  += $2 }
  77 $1 == "Cached:"       { memfree  += $2 }
  78 $1 == "Buffers:"      { memfree  += $2 }
  79 $1 == "MemTotal:"     { memtotal  = $2 }
  80 $1 == "SwapFree:"     { swapfree  = $2 }
  81 $1 == "SwapTotal:"    { swaptotal = $2 }
  82 END {
  83     if (memavail != 0) { memfree = memavail ; }
  84     print int((memtotal -  memfree)  / memtotal * 100),
  85           int((swaptotal - swapfree) / swaptotal * 100)
  86 }')
  87     _mem_usage="$1"
  88     _swap_usage="$2"
  89
  90     # Shutdown CTDB when memory is below the configured limit
  91     if [ -n "$CTDB_MONITOR_FREE_MEMORY" ] ; then
  92         if [ $_mem_usage -ge $CTDB_MONITOR_FREE_MEMORY ] ; then
  93             echo "CRITICAL: OOM - ${_mem_usage}% usage >= ${CTDB_MONITOR_FREE_MEMORY}% (CTDB threshold)"
  94             echo "CRITICAL: Shutting down CTDB!!!"
  95             echo "$_meminfo"
  96             ps auxfww
  97             set_proc "sysrq-trigger" "m"
  98             ctdb disable
  99             sleep 3
 100             ctdb shutdown
 101         fi
 102     fi
 103
 104     # Warn when low on memory
 105     if [ -n "$CTDB_MONITOR_FREE_MEMORY_WARN" ] ; then
 106         if [ $_mem_usage -ge $CTDB_MONITOR_FREE_MEMORY_WARN ] ; then
 107             echo "WARNING: memory usage is excessive - ${_mem_usage}% >=  ${CTDB_MONITOR_FREE_MEMORY_WARN}% (CTDB threshold)"
 108         fi
 109     fi
 110
 111     # We should never enter swap, so SwapTotal == SwapFree.
 112     if [ "$CTDB_CHECK_SWAP_IS_NOT_USED" = "yes" ] ; then
 113         if [ $_swap_usage -gt 0 ] ; then
 114             echo We are swapping:
 115             echo "$_meminfo"
 116             ps auxfww
 117         fi
 118     fi
 119 }
 120
 121
 122 case "$1" in
 123     monitor)
 124         monitor_filesystem_usage
 125         monitor_memory_usage
 126         ;;
 127
 128     *)
 129         ctdb_standard_event_handler "$@"
 130         ;;
 131 esac
 132
 133 exit 0