From 4164d7bf3153a2fd9081b4d073bfa88fec1507ad Mon Sep 17 00:00:00 2001
From: Martin Schwenke <martin@meltin.net>
Date: Tue, 18 Aug 2015 15:22:23 +1000
Subject: [PATCH] ctdb-scripts: Add default filesystem usage warnings

Always check filesystem usage for the database directories.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

Autobuild-User(master): Amitay Isaacs <amitay@samba.org>
Autobuild-Date(master): Sat Aug 29 20:08:48 CEST 2015 on sn-devel-104
---
 ctdb/config/events.d/05.system                   | 22 ++++++++++++++++++++++
 ctdb/doc/ctdbd.conf.5.xml                        |  6 +++++-
 ctdb/tests/eventscripts/05.system.monitor.001.sh |  4 +++-
 ctdb/tests/eventscripts/scripts/local.sh         |  5 ++++-
 ctdb/tests/eventscripts/stubs/df                 | 11 +++++++++++
 5 files changed, 45 insertions(+), 3 deletions(-)

diff --git a/ctdb/config/events.d/05.system b/ctdb/config/events.d/05.system
index c305c4b1548..69fcec2e854 100755
--- a/ctdb/config/events.d/05.system
+++ b/ctdb/config/events.d/05.system
@@ -67,8 +67,30 @@ check_thresholds ()
     fi
 }
 
+set_monitor_filsystem_usage_defaults ()
+{
+    _fs_defaults_cache="${service_state_dir}/cache_monitor_filsystem_usage_defaults"
+
+    if [ ! -r "$_fs_defaults_cache" ] ; then
+	# Determine filesystem for each database directory, generate
+	# an entry to warn at 90%, de-duplicate entries, put all items
+	# on 1 line (so the read below gets everything)
+	for _t in "${CTDB_DBDIR:-${CTDB_VARDIR}}" \
+		      "${CTDB_DBDIR_PERSISTENT:-${CTDB_VARDIR}/persistent}" \
+		      "${CTDB_DBDIR_STATE:-${CTDB_VARDIR}/state}" ; do
+	    df -kP "$_t" | awk 'NR == 2 { printf "%s:90\n", $6 }'
+	done | sort -u | xargs >"$_fs_defaults_cache"
+    fi
+
+    read CTDB_MONITOR_FILESYSTEM_USAGE <"$_fs_defaults_cache"
+}
+
 monitor_filesystem_usage ()
 {
+    if [ -z "$CTDB_MONITOR_FILESYSTEM_USAGE" ] ; then
+	set_monitor_filsystem_usage_defaults
+    fi
+
     # Check each specified filesystem, specified in format
     # <fs_mount>:<fs_warn_threshold>[:fs_unhealthy_threshold]
     for _fs in $CTDB_MONITOR_FILESYSTEM_USAGE ; do
diff --git a/ctdb/doc/ctdbd.conf.5.xml b/ctdb/doc/ctdbd.conf.5.xml
index e04c89aafab..f45c724f4b5 100644
--- a/ctdb/doc/ctdbd.conf.5.xml
+++ b/ctdb/doc/ctdbd.conf.5.xml
@@ -1321,7 +1321,11 @@ CTDB_PER_IP_ROUTING_TABLE_ID_HIGH=9000
 	    </para>
 
 	    <para>
-	      No default.
+	      Default is to warn for each filesystem containing a
+	      database directory (<envar>CTDB_DBDIR</envar>,
+	      <envar>CTDB_DBDIR_PERSISTENT</envar>,
+	      <envar>CTDB_DBDIR_STATE</envar>) with a threshold of
+	      90%.
 	    </para>
 	  </listitem>
 	</varlistentry>
diff --git a/ctdb/tests/eventscripts/05.system.monitor.001.sh b/ctdb/tests/eventscripts/05.system.monitor.001.sh
index 5d513b6de41..d88d0e3edd9 100755
--- a/ctdb/tests/eventscripts/05.system.monitor.001.sh
+++ b/ctdb/tests/eventscripts/05.system.monitor.001.sh
@@ -8,5 +8,7 @@ setup_memcheck
 
 CTDB_MONITOR_FILESYSTEM_USAGE=""
 setup_fscheck 100
-ok_null
+ok <<EOF
+WARNING: Filesystem ${CTDB_DBDIR} utilization 100% >= threshold 90%
+EOF
 simple_test
diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh
index 57e022536cd..6f3c34e17f6 100644
--- a/ctdb/tests/eventscripts/scripts/local.sh
+++ b/ctdb/tests/eventscripts/scripts/local.sh
@@ -112,7 +112,10 @@ setup_generic ()
 
 
     export CTDB_DBDIR="${EVENTSCRIPTS_TESTS_VAR_DIR}/db"
-    mkdir -p "${CTDB_DBDIR}/persistent"
+    export CTDB_DBDIR_PERSISTENT="${CTDB_DBDIR}/persistent"
+    export CTDB_DBDIR_STATE="${CTDB_DBDIR}/state"
+    mkdir -p "$CTDB_DBDIR_PERSISTENT"
+    mkdir -p "$CTDB_DBDIR_STATE"
 
     export FAKE_TDBTOOL_SUPPORTS_CHECK="yes"
     export FAKE_TDB_IS_OK
diff --git a/ctdb/tests/eventscripts/stubs/df b/ctdb/tests/eventscripts/stubs/df
index c74288f1489..eb91a4c851d 100755
--- a/ctdb/tests/eventscripts/stubs/df
+++ b/ctdb/tests/eventscripts/stubs/df
@@ -17,6 +17,17 @@ fi
 
 fs="$1"
 
+# Anything starting with CTDB_DBDIR gets canonicalised to CTDB_DBDIR.
+# This helps with the setting of defaults for the filesystem checks.
+if [ "${fs#${CTDB_DBDIR}}" != "$fs" ] ; then
+    fs="$CTDB_DBDIR"
+fi
+
+# A default, for tests that don't initialise this...
+if [ -z "$FAKE_FS_USE" ] ; then
+    FAKE_FS_USE=10
+fi
+
 echo "Filesystem             1024-blocks      Used Available Capacity Mounted on"
 
 blocks="1000000"
-- 
2.11.4.GIT