ctdb/config/events/legacy/05.system.script

   1 #!/bin/sh
   2 # ctdb event script for checking local file system utilization
   3
   4 [ -n "$CTDB_BASE" ] ||
   5         CTDB_BASE=$(d=$(dirname "$0") && cd -P "$d" && dirname "$PWD")
   6
   7 . "${CTDB_BASE}/functions"
   8
   9 load_script_options
  10
  11 ctdb_setup_state_dir "service" "system-monitoring"
  12
  13 validate_percentage()
  14 {
  15         case "$1" in
  16         "") return 1 ;; # A failure that doesn't need a warning
  17         [0-9] | [0-9][0-9] | 100) return 0 ;;
  18         *)
  19                 echo "WARNING: ${1} is an invalid percentage in \"${2}\" check"
  20                 return 1
  21                 ;;
  22         esac
  23 }
  24
  25 check_thresholds()
  26 {
  27         _thing="$1"
  28         _thresholds="$2"
  29         _usage="$3"
  30         _unhealthy_callout="$4"
  31
  32         case "$_thresholds" in
  33         *:*)
  34                 _warn_threshold="${_thresholds%:*}"
  35                 _unhealthy_threshold="${_thresholds#*:}"
  36                 ;;
  37         *)
  38                 _warn_threshold="$_thresholds"
  39                 _unhealthy_threshold=""
  40                 ;;
  41         esac
  42
  43         _t=$(echo "$_thing" | sed -e 's@/@SLASH_@g' -e 's@ @_@g')
  44         # script_state_dir set by ctdb_setup_state_dir()
  45         # shellcheck disable=SC2154
  46         _cache="${script_state_dir}/cache_${_t}"
  47         if validate_percentage "$_unhealthy_threshold" "$_thing"; then
  48                 if [ "$_usage" -ge "$_unhealthy_threshold" ]; then
  49                         printf 'ERROR: %s utilization %d%% >= threshold %d%%\n' \
  50                                "$_thing" \
  51                                "$_usage" \
  52                                "$_unhealthy_threshold"
  53                         eval "$_unhealthy_callout"
  54                         echo "$_usage" >"$_cache"
  55                         exit 1
  56                 fi
  57         fi
  58
  59         if validate_percentage "$_warn_threshold" "$_thing"; then
  60                 if [ "$_usage" -ge "$_warn_threshold" ]; then
  61                         if [ -r "$_cache" ]; then
  62                                 read -r _prev <"$_cache"
  63                         else
  64                                 _prev=""
  65                         fi
  66                         if [ "$_usage" = "$_prev" ]; then
  67                                 return
  68                         fi
  69                         printf 'WARNING: %s utilization %d%% >= threshold %d%%\n' \
  70                                "$_thing" \
  71                                "$_usage" \
  72                                "$_warn_threshold"
  73                         echo "$_usage" >"$_cache"
  74                 else
  75                         if [ ! -r "$_cache" ]; then
  76                                 return
  77                         fi
  78                         printf 'NOTICE: %s utilization %d%% < threshold %d%%\n' \
  79                                "$_thing" \
  80                                "$_usage" \
  81                                "$_warn_threshold"
  82                         rm -f "$_cache"
  83                 fi
  84         fi
  85 }
  86
  87 set_monitor_filsystem_usage_defaults()
  88 {
  89         _fs_defaults_cache="${script_state_dir}/cache_filsystem_usage_defaults"
  90
  91         if [ ! -r "$_fs_defaults_cache" ]; then
  92                 # Determine filesystem for each database directory, generate
  93                 # an entry to warn at 90%, de-duplicate entries, put all items
  94                 # on 1 line (so the read below gets everything)
  95                 for _t in "${CTDB_DBDIR:-${CTDB_VARDIR}}" \
  96                         "${CTDB_DBDIR_PERSISTENT:-${CTDB_VARDIR}/persistent}" \
  97                         "${CTDB_DBDIR_STATE:-${CTDB_VARDIR}/state}"; do
  98                         df -kP "$_t" | awk 'NR == 2 { printf "%s:90\n", $6 }'
  99                 done | sort -u | xargs >"$_fs_defaults_cache"
 100         fi
 101
 102         read -r CTDB_MONITOR_FILESYSTEM_USAGE <"$_fs_defaults_cache"
 103 }
 104
 105 monitor_filesystem_usage()
 106 {
 107         if [ -z "$CTDB_MONITOR_FILESYSTEM_USAGE" ]; then
 108                 set_monitor_filsystem_usage_defaults
 109         fi
 110
 111         # Check each specified filesystem, specified in format
 112         # <fs_mount>:<fs_warn_threshold>[:fs_unhealthy_threshold]
 113         for _fs in $CTDB_MONITOR_FILESYSTEM_USAGE; do
 114                 _fs_mount="${_fs%%:*}"
 115                 _fs_thresholds="${_fs#*:}"
 116
 117                 if [ ! -d "$_fs_mount" ]; then
 118                         echo "WARNING: Directory ${_fs_mount} does not exist"
 119                         continue
 120                 fi
 121
 122                 # Get current utilization
 123                 _fs_usage=$(df -kP "$_fs_mount" |
 124                         sed -n -e 's@.*[[:space:]]\([[:digit:]]*\)%.*@\1@p')
 125                 if [ -z "$_fs_usage" ]; then
 126                         printf 'WARNING: Unable to get FS utilization for %s\n' \
 127                                "$_fs_mount"
 128                         continue
 129                 fi
 130
 131                 check_thresholds "Filesystem ${_fs_mount}" \
 132                         "$_fs_thresholds" \
 133                         "$_fs_usage"
 134         done
 135 }
 136
 137 dump_memory_info()
 138 {
 139         get_proc "meminfo"
 140         ps auxfww
 141         set_proc "sysrq-trigger" "m"
 142 }
 143
 144 monitor_memory_usage()
 145 {
 146         # Defaults
 147         if [ -z "$CTDB_MONITOR_MEMORY_USAGE" ]; then
 148                 CTDB_MONITOR_MEMORY_USAGE=80
 149         fi
 150
 151         _meminfo=$(get_proc "meminfo")
 152         # Intentional word splitting here
 153         # shellcheck disable=SC2046
 154         set -- $(echo "$_meminfo" | awk '
 155 $1 == "MemAvailable:" { memavail += $2 }
 156 $1 == "MemFree:"      { memfree  += $2 }
 157 $1 == "Cached:"       { memfree  += $2 }
 158 $1 == "Buffers:"      { memfree  += $2 }
 159 $1 == "MemTotal:"     { memtotal  = $2 }
 160 $1 == "SwapFree:"     { swapfree  = $2 }
 161 $1 == "SwapTotal:"    { swaptotal = $2 }
 162 END {
 163     if (memavail != 0) { memfree = memavail ; }
 164     if (memtotal + swaptotal != 0) {
 165         usedtotal = memtotal - memfree + swaptotal - swapfree
 166         print int(usedtotal / (memtotal + swaptotal) * 100)
 167     } else {
 168         print 0
 169     }
 170 }')
 171         _mem_usage="$1"
 172
 173         check_thresholds "System memory" \
 174                 "$CTDB_MONITOR_MEMORY_USAGE" \
 175                 "$_mem_usage" \
 176                 dump_memory_info
 177 }
 178
 179 case "$1" in
 180 monitor)
 181         # Load/cache database options from configuration file
 182         ctdb_get_db_options
 183
 184         rc=0
 185         monitor_filesystem_usage || rc=$?
 186         monitor_memory_usage || rc=$?
 187         exit $rc
 188         ;;
 189 esac
 190
 191 exit 0