From 33683d9fd2b3d6d240c539e50e61af0fca0fe5aa Mon Sep 17 00:00:00 2001
From: Josh Triplett <josh@freedesktop.org>
Date: Sun, 22 Jul 2007 20:27:57 -0700
Subject: [PATCH] Fix test-suite to handle stdout and stderr separately, and
 fix up tests

test-suite merged stdout and stderr, which relied on the ordering of data from
the two streams in the merged stream.  This made test-suite frequently fail on
tests with both output and errors, when the ordering didn't happen to match
what the test assumed.  Handle each of the streams separately, and update the
tests accordingly.

Also clean up the output of "test-suite format" to avoid outputting values
equal to the defaults.

Signed-off-by: Josh Triplett <josh@freedesktop.org>
---
 Makefile                                      |  9 +++-
 validation/address_space.c                    |  5 +--
 validation/bad-array-designated-initializer.c |  8 +---
 validation/bad-assignment.c                   |  8 +---
 validation/cond_expr2.c                       |  5 +--
 validation/label-asm.c                        |  1 -
 validation/preprocessor/preprocessor18.c      |  8 ++--
 validation/preprocessor/preprocessor19.c      |  6 ++-
 validation/preprocessor/preprocessor21.c      |  6 +--
 validation/struct-as.c                        |  1 -
 validation/test-suite                         | 61 ++++++++++++---------------
 11 files changed, 54 insertions(+), 64 deletions(-)

diff --git a/Makefile b/Makefile
index d306f5b1..7e754b9b 100644
--- a/Makefile
+++ b/Makefile
@@ -188,5 +188,10 @@ check: all
 	$(Q)cd validation && ./test-suite
 
 clean-check:
-	find validation/ -name "*.c.[egd]*" -exec rm {} \;
-
+	find validation/ \( -name "*.c.output.expected" \
+	                 -o -name "*.c.output.got" \
+	                 -o -name "*.c.output.diff" \
+	                 -o -name "*.c.error.expected" \
+	                 -o -name "*.c.error.got" \
+	                 -o -name "*.c.error.diff" \
+	                 \) -exec rm {} \;
diff --git a/validation/address_space.c b/validation/address_space.c
index 79c1d55c..2d06be7d 100644
--- a/validation/address_space.c
+++ b/validation/address_space.c
@@ -8,11 +8,10 @@ static int sys_do_stuff(void __user *user_addr)
 }
 /*
  * check-name: address_space attribute
- * check-command: sparse $file
  *
- * check-output-start
+ * check-error-start
 address_space.c:7:21: warning: incorrect type in argument 1 (different address spaces)
 address_space.c:7:21:    expected void *addr
 address_space.c:7:21:    got void <asn:1>*user_addr
- * check-output-end
+ * check-error-end
  */
diff --git a/validation/bad-array-designated-initializer.c b/validation/bad-array-designated-initializer.c
index ecc5bd3b..cde07854 100644
--- a/validation/bad-array-designated-initializer.c
+++ b/validation/bad-array-designated-initializer.c
@@ -4,14 +4,10 @@ static int a[] = {
 };
 /*
  * check-name: Bad array designated initializer
- * check-command: sparse $file
- * check-exit-value: 1
  *
- * check-output-start
+ * check-error-start
 bad-array-designated-initializer.c:3:3: error: Expected constant expression
 bad-array-designated-initializer.c:3:3: error: Expected } at end of initializer
 bad-array-designated-initializer.c:3:3: error: got \
- * check-output-end
- *
- * check-known-to-fail
+ * check-error-end
  */
diff --git a/validation/bad-assignment.c b/validation/bad-assignment.c
index 66f6fe83..327f7da2 100644
--- a/validation/bad-assignment.c
+++ b/validation/bad-assignment.c
@@ -6,13 +6,9 @@ static int foo(int a)
 }
 /*
  * check-name: bad assignment
- * check-command: sparse $file
- * check-exit-value: 1
  *
- * check-output-start
+ * check-error-start
 bad-assignment.c:3:6: error: Expected ; at end of statement
 bad-assignment.c:3:6: error: got \
- * check-output-end
- *
- * check-known-to-fail
+ * check-error-end
  */
diff --git a/validation/cond_expr2.c b/validation/cond_expr2.c
index e53cd133..174f2d17 100644
--- a/validation/cond_expr2.c
+++ b/validation/cond_expr2.c
@@ -10,14 +10,13 @@ static void f(void)
 /*
  * check-name: type of conditional expression
  * check-description: Used to miss qualifier mixing and mishandle void *
- * check-command: sparse $file
  *
- * check-output-start
+ * check-error-start
 cond_expr2.c:6:4: warning: incorrect type in assignment (different modifiers)
 cond_expr2.c:6:4:    expected void volatile *extern [addressable] [toplevel] q
 cond_expr2.c:6:4:    got void const volatile *
 cond_expr2.c:8:4: warning: incorrect type in assignment (different modifiers)
 cond_expr2.c:8:4:    expected int volatile *extern [addressable] [toplevel] [assigned] r
 cond_expr2.c:8:4:    got int const volatile *
- * check-output-end
+ * check-error-end
  */
diff --git a/validation/label-asm.c b/validation/label-asm.c
index 40cfc845..411020ac 100644
--- a/validation/label-asm.c
+++ b/validation/label-asm.c
@@ -9,5 +9,4 @@ l:
 /*
  * check-name: Label followed by __asm__
  * check-description: Sparse used to parse the __asm__ as modifying the label.
- * check-command: sparse $file
  */
diff --git a/validation/preprocessor/preprocessor18.c b/validation/preprocessor/preprocessor18.c
index b4de1e68..20169e8b 100644
--- a/validation/preprocessor/preprocessor18.c
+++ b/validation/preprocessor/preprocessor18.c
@@ -4,14 +4,14 @@
 /*
  * check-name: Preprocessor #18
  * check-command: sparse -E $file
- * check-exit-value: 1
  *
  * check-output-start
 
-preprocessor/preprocessor18.c:2:2: error: expected identifier to 'define'
-preprocessor/preprocessor18.c:3:2: error: expected identifier to 'undef'
 
  * check-output-end
  *
- * check-known-to-fail
+ * check-error-start
+preprocessor/preprocessor18.c:2:2: error: expected identifier to 'define'
+preprocessor/preprocessor18.c:3:2: error: expected identifier to 'undef'
+ * check-error-end
  */
diff --git a/validation/preprocessor/preprocessor19.c b/validation/preprocessor/preprocessor19.c
index ef27740d..7dd3820d 100644
--- a/validation/preprocessor/preprocessor19.c
+++ b/validation/preprocessor/preprocessor19.c
@@ -8,9 +8,11 @@ A
  * check-command: sparse -E $file
  *
  * check-output-start
-preprocessor/preprocessor19.c:4:9: warning: preprocessor token A redefined
-preprocessor/preprocessor19.c:3:9: this was the original definition
 
 y
  * check-output-end
+ * check-error-start
+preprocessor/preprocessor19.c:4:9: warning: preprocessor token A redefined
+preprocessor/preprocessor19.c:3:9: this was the original definition
+ * check-error-end
  */
diff --git a/validation/preprocessor/preprocessor21.c b/validation/preprocessor/preprocessor21.c
index 1719fc07..4b55a6b8 100644
--- a/validation/preprocessor/preprocessor21.c
+++ b/validation/preprocessor/preprocessor21.c
@@ -4,13 +4,13 @@
  * check-name: Preprocessor #21
  * check-description: This used to hang Sparse.
  * check-command: sparse -E $file
- * check-exit-value: 1
  *
  * check-output-start
-preprocessor/preprocessor21.c:2:2: error: unterminated preprocessor conditional
 
 
  * check-output-end
  *
- * check-known-to-fail
+ * check-error-start
+preprocessor/preprocessor21.c:2:2: error: unterminated preprocessor conditional
+ * check-error-end
  */
diff --git a/validation/struct-as.c b/validation/struct-as.c
index cafbaaeb..f31f7c96 100644
--- a/validation/struct-as.c
+++ b/validation/struct-as.c
@@ -16,5 +16,4 @@ static int broken(struct hello __user *sp)
 }
 /*
  * check-name: Address space of a struct member
- * check-command: sparse $file
  */
diff --git a/validation/test-suite b/validation/test-suite
index 6b9d1b71..1a357030 100755
--- a/validation/test-suite
+++ b/validation/test-suite
@@ -116,18 +116,22 @@ do_test()
 
 	# grab the expected output
 	sed -n '/check-output-start/,/check-output-end/p' $file \
-		| grep -v check-output > "$file".expected
+		| grep -v check-output > "$file".output.expected
+	sed -n '/check-error-start/,/check-error-end/p' $file \
+		| grep -v check-error > "$file".error.expected
 
 	# grab the actual output & exit value
-	$cmd 1> $file.got 2>&1
+	$cmd 1> $file.output.got 2> $file.error.got
 	actual_exit_value=$?
 
-	diff -u "$file".expected "$file".got > "$file".diff
-	if [ "$?" -ne "0" ]; then
-		error "actual output does not match the expected one."
-		error  "see $file.* for further investigation."
-		test_failed=1
-	fi
+	for stream in output error; do
+		diff -u "$file".$stream.expected "$file".$stream.got > "$file".$stream.diff
+		if [ "$?" -ne "0" ]; then
+			error "actual $stream text does not match expected $stream text."
+			error  "see $file.$stream.* for further investigation."
+			test_failed=1
+		fi
+	done
 
 	if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
 		error "Actual exit value does not match the expected one."
@@ -177,36 +181,27 @@ do_format()
 	fi
 	file="$1"
 	cmd=`eval echo $default_path/$fcmd`
-	$cmd 1> $file.got 2>&1
+	$cmd 1> $file.output.got 2> $file.error.got
 	fexit_value=$?
-	#foutput=`sed -e "s/^\(.*\)/ * check-output:\1/" $file.got`
-	foutput=`cat $file.got`
-	if [ -z "$foutput" ]; then
-	format=`cat <<_EOF
+	cat <<_EOF
 /*
  * check-name: $fname
- *
- * check-command: $fcmd
- * check-exit-value: $fexit_value
- */
 _EOF
-`
-	else
-	format=`cat <<_EOF
-/*
- * check-name: $fname
- *
- * check-command: $fcmd
- * check-exit-value: $fexit_value
- *
- * check-output-start
-$foutput
- * check-output-end
- */
-_EOF
-`
+	if [ "$fcmd" != "$default_cmd" ]; then
+		echo " * check-command: $fcmd"
 	fi
-	echo "$format"
+	if [ "$fexit_value" -ne "0" ]; then
+		echo " * check-exit-value: $fexit_value"
+	fi
+	for stream in output error; do
+		if [ -s "$file.$stream.got" ]; then
+			echo " *"
+			echo " * check-$stream-start"
+			cat "$file.$stream.got"
+			echo " * check-$stream-end"
+		fi
+	done
+	echo " */"
 	return 0
 }
 
-- 
2.11.4.GIT