From c8ce0cace259527c0183b8e654ad664665e08e49 Mon Sep 17 00:00:00 2001
From: "Kyle J. McKay" <mackyle@gmail.com>
Date: Sun, 28 Aug 2016 23:25:21 -0700
Subject: [PATCH] bang: detect SIGINT and SIGTERM failures

Detect bang failures because the process was killed with SIGINT or
SIGTERM and count those as a failure.

This way updates that start failing because they are now hanging every
time (and then being killed by jobd.pl) will be caught and generate a
failure message after the minimum count/interval is reached rather than
silently continuing to fail with no notification.

Previously this detection was undesirable as it would have likely
resulted in additional spurious failed+recovery messages.  However, now
that there is a minimum failure count and/or interval required including
SIGINT and SIGTERM kills as failures will no longer cause that problem.

Signed-off-by: Kyle J. McKay <mackyle@gmail.com>
---
 shlib.sh | 39 +++++++++++++++++++++++++++++----------
 1 file changed, 29 insertions(+), 10 deletions(-)

diff --git a/shlib.sh b/shlib.sh
index bdf8a16..fa9610f 100644
--- a/shlib.sh
+++ b/shlib.sh
@@ -276,29 +276,48 @@ mail() {
 # set bang_action to string of the failed action ('clone', 'update', ...);
 # re-define the bang_trap() function to do custom cleanup before bailing out
 bang() {
+	bang_active=1
+	bang_cmd="$*"
+	bang_errcode=0
 	if [ -n "$show_progress" ]; then
 		exec 3>&1
-		errcode=
-		read -r errcode <<-EOT || :
+		read -r bang_errcode <<-EOT || :
 		$(
 			exec 4>&3 3>&1 1>&4 4>&-
 			{ "$@" 3>&- || echo $? >&3; } 2>&1 | tee -i -a "$bang_log"
 		)
 		EOT
 		exec 3>&-
-		if [ -z "$errcode" ]; then
+		if [ -z "$bang_errcode" ] || [ "$bang_errcode" = "0" ]; then
 			# All right. Cool.
+			bang_active=
+			bang_cmd=
 			return;
 		fi
 	else
 		if "$@" >>"$bang_log" 2>&1; then
 			# All right. Cool.
+			bang_active=
+			bang_cmd=
 			return;
 		else
-			errcode="$?"
+			bang_errcode="$?"
 		fi
 	fi
+	bang_failed
+}
+
+bang_failed() {
+	bang_active=
 	unset GIT_DIR
+	touch .banged
+	cat "$bang_log" > .banglog
+	echo "" >> .banglog
+	echo "$bang_cmd failed with error code $bang_errcode" >> .banglog
+	if [ -n "$show_progress" ]; then
+		echo ""
+		echo "$bang_cmd failed with error code $bang_errcode"
+	fi
 	if [ -e .bangagain ]; then
 		git config --remove-section girocco.bang 2>/dev/null || :
 		rm -f .bangagain
@@ -322,7 +341,7 @@ bang() {
 		[ $bangcount -le 1 ] || rsubj=" repeatedly"
 		[ -z "$bangaddrs" ] ||
 		{
-			echo "$* failed with error code $errcode"
+			echo "$bang_cmd failed with error code $bang_errcode"
 			echo ""
 			rsubj=
 			if [ $bangcount -gt 1 ]; then
@@ -338,10 +357,9 @@ bang() {
 		} | mail -s "[$cfg_name] $proj $bang_action failed$rsubj" "$bangaddrs"
 		git config --bool girocco.bang.messagesent true
 	fi
-	touch .banged
-	cat "$bang_log" > .banglog
 	bang_trap
-	exit 1
+	[ -n "$bang_errcode" ] && [ "$bang_errcode" != "0" ] || bang_errcode=1
+	exit $bang_errcode
 }
 
 # bang_eval CMD... will evaluate the command with well-defined failure mode;
@@ -352,6 +370,7 @@ bang_eval() {
 
 # Default bang settings:
 bang_setup() {
+	bang_active=
 	bang_action="lame_programmer"
 	bang_trap() { :; }
 	bang_log="$(mktemp -t repomgr-XXXXXX)"
@@ -360,8 +379,8 @@ bang_setup() {
 		exit 1
 	}
 	trap 'rm -f "$bang_log"' EXIT
-	trap 'exit 130' INT
-	trap 'exit 143' TERM
+	trap '[ -z "$bang_active" ] || { bang_errcode=130; bang_failed; }; exit 130' INT
+	trap '[ -z "$bang_active" ] || { bang_errcode=143; bang_failed; }; exit 143' TERM
 }
 
 # Remove banged status
-- 
2.11.4.GIT