ctdb/tests/eventscripts/05.system.monitor.018.sh

   1 #!/bin/sh
   2
   3 . "${TEST_SCRIPTS_DIR}/unit.sh"
   4
   5 define_test "Check throttling of warnings"
   6
   7 CTDB_MONITOR_MEMORY_USAGE="70:80"
   8 CTDB_MONITOR_SWAP_USAGE=""
   9
  10 # Below threshold, nothing logged
  11 setup_memcheck 67 0
  12 ok_null
  13 simple_test
  14
  15 setup_memcheck 71 0
  16 ok "WARNING: System memory utilization 71% >= threshold 70%"
  17 simple_test
  18
  19 # 2nd time at same level, nothing logged
  20 setup_memcheck 71 0
  21 ok_null
  22 simple_test
  23
  24 setup_memcheck 73 0
  25 ok "WARNING: System memory utilization 73% >= threshold 70%"
  26 simple_test
  27
  28 # 2nd time at same level, nothing logged
  29 setup_memcheck 73 0
  30 ok_null
  31 simple_test
  32
  33 setup_memcheck 79 0
  34 ok "WARNING: System memory utilization 79% >= threshold 70%"
  35 simple_test
  36
  37 setup_memcheck 80 0
  38 required_result 1 <<EOF
  39 ERROR: System memory utilization 80% >= threshold 80%
  40 MemTotal:        3940712 kB
  41 MemFree:          225268 kB
  42 Buffers:          146120 kB
  43 Cached:          416754 kB
  44 SwapCached:        56016 kB
  45 Active:          2422104 kB
  46 Inactive:        1019928 kB
  47 Active(anon):    1917580 kB
  48 Inactive(anon):   523080 kB
  49 Active(file):     504524 kB
  50 Inactive(file):   496848 kB
  51 Unevictable:        4844 kB
  52 Mlocked:            4844 kB
  53 SwapTotal:       5857276 kB
  54 SwapFree:        5857276 kB
  55 ...
  56 USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
  57 root         2  0.0  0.0      0     0 ?        S    Aug28   0:00 [kthreadd]
  58 root         3  0.0  0.0      0     0 ?        S    Aug28   0:43  \_ [ksoftirqd/0]
  59 ...
  60 root         1  0.0  0.0   2976   624 ?        Ss   Aug28   0:07 init [2]
  61 root       495  0.0  0.0   3888  1640 ?        Ss   Aug28   0:00 udevd --daemon
  62 ...
  63 [MORE FAKE ps OUTPUT]
  64 EOF
  65 simple_test
  66
  67 # Fall back into warning at same level as last warning... should log
  68 setup_memcheck 79 0
  69 ok "WARNING: System memory utilization 79% >= threshold 70%"
  70 simple_test
  71
  72 # Below threshold, notice
  73 setup_memcheck 69 0
  74 ok <<EOF
  75 NOTICE: System memory utilization 69% < threshold 70%
  76 EOF
  77 simple_test
  78
  79 # Further reduction, nothing logged
  80 setup_memcheck 68 0
  81 ok_null
  82 simple_test
  83
  84 # Back up into warning at same level as last warning... should log
  85 setup_memcheck 79 0
  86 ok "WARNING: System memory utilization 79% >= threshold 70%"
  87 simple_test
  88
  89 # Back up above critical threshold... unhealthy
  90 setup_memcheck 81 0
  91 required_result 1 <<EOF
  92 ERROR: System memory utilization 81% >= threshold 80%
  93 MemTotal:        3940712 kB
  94 MemFree:          225268 kB
  95 Buffers:          146120 kB
  96 Cached:          377347 kB
  97 SwapCached:        56016 kB
  98 Active:          2422104 kB
  99 Inactive:        1019928 kB
 100 Active(anon):    1917580 kB
 101 Inactive(anon):   523080 kB
 102 Active(file):     504524 kB
 103 Inactive(file):   496848 kB
 104 Unevictable:        4844 kB
 105 Mlocked:            4844 kB
 106 SwapTotal:       5857276 kB
 107 SwapFree:        5857276 kB
 108 ...
 109 USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
 110 root         2  0.0  0.0      0     0 ?        S    Aug28   0:00 [kthreadd]
 111 root         3  0.0  0.0      0     0 ?        S    Aug28   0:43  \_ [ksoftirqd/0]
 112 ...
 113 root         1  0.0  0.0   2976   624 ?        Ss   Aug28   0:07 init [2]
 114 root       495  0.0  0.0   3888  1640 ?        Ss   Aug28   0:00 udevd --daemon
 115 ...
 116 [MORE FAKE ps OUTPUT]
 117 EOF
 118 simple_test
 119
 120 # Straight back down to a good level... notice
 121 setup_memcheck 65 0
 122 ok "NOTICE: System memory utilization 65% < threshold 70%"
 123 simple_test