contrib/dg-cmp-results.sh

   1 #!/bin/bash
   2 # Copyright (C) 2006-2023 Free Software Foundation, Inc.
   3 #
   4 # Analyze changes in GCC DejaGNU test logs for binutils, gcc, gdb, etc.
   5 # Original version written in 2005 by James Lemke <jwlemke@wasabisystems.com>.
   6 #
   7 # See usage() below.
   8
   9 usage () {
  10     cat <<EOF >&2
  11 Usage:
  12     dg-cmp-results.sh [-v] [-v] [-v] <variant-name> <old-file> <new-file>
  13     <variant-name> names the desired variant, "/" must be written as "\/".
  14     Use the empty string ("") for the first variant in each file.
  15     Output is to stdout.
  16     Non-verbose output is degradation info like PASS->FAIL.
  17     -v adds improvement info like FAIL->PASS.
  18     -v -v adds info like tests that are no longer run.
  19     -v -v -v adds info for tests that have not changed status.
  20     -v -v -v -v is used for debugging.
  21 EOF
  22 }
  23
  24 verbose=0
  25 while test "$1" = "-v"; do
  26     verbose=`expr $verbose + 1`
  27     shift
  28 done
  29
  30 if test $# -ne 3 ; then
  31     usage
  32     exit 1
  33 fi
  34
  35 if test ! -f "$2"; then
  36     echo "unable to open $2" >&2
  37     exit 1
  38 fi
  39
  40 if test ! -f "$3"; then
  41     echo "unable to open $3" >&2
  42     exit 1
  43 fi
  44
  45 # Command differences for various platforms.
  46 case `uname -s` in
  47 Darwin|NetBSD)
  48     E=-E        # sed
  49     ;;
  50 *)
  51     E=-r        # sed
  52     ;;
  53 esac
  54
  55 # sections are identified by separator lines beginning with '\t\t==='.
  56 # section 0 identifies run date, target, and host.
  57 # section 1 and subsequent contain test data for a target variant.
  58 # -skip to /^Running target/ and use that line to identify the variant.
  59 # -subsequent lines contain the result data.  They begin with:
  60 # '(PASS|FAIL|XFAIL|XPASS|UNTESTED|UNSUPPORTED|UNRESOLVED):'
  61 VARIANT="$1"
  62 OFILE="$2"
  63 OBASE=`basename "$2"`
  64 NFILE="$3"
  65 NBASE=`basename "$3"`
  66 TMPDIR=${TMPDIR:-/tmp}
  67
  68 echo "dg-cmp-results.sh: Verbosity is ${verbose}, Variant is \"${VARIANT}\""
  69 echo
  70
  71 header="^Running target $VARIANT"
  72
  73 temp=`grep "$header" $OFILE`
  74 if test -z "$temp"; then
  75     echo "Error: variant \"$VARIANT\" not found in $OFILE."
  76     exit 1
  77 fi
  78 temp=`grep "$header" $NFILE`
  79 if test -z "$temp"; then
  80     echo "Error: variant \"$VARIANT\" not found in $NFILE."
  81     exit 1
  82 fi
  83 unset temp
  84
  85 # Copy out the old file's section 0.
  86 echo "Older log file: $OFILE"
  87 sed $E -e '/^[[:space:]]+===/,$d' $OFILE
  88
  89 # Copy out the new file's section 0.
  90 echo "Newer log file: $NFILE"
  91 sed $E -e '/^[[:space:]]+===/,$d' $NFILE
  92
  93 # Create a temporary file from the old file's interesting section.
  94 sed $E -e "/$header/,/^[[:space:]]+===.*Summary ===/!d" \
  95   -e '/^[A-Z]+:/!d' \
  96   -e '/^(WARNING|ERROR):/d' \
  97   -e 's/\r$//' \
  98   -e 's/^/O:/' \
  99   $OFILE |
 100   sort -s -t : -k 3b - \
 101   >$TMPDIR/o$$-$OBASE
 102
 103 # Create a temporary file from the new file's interesting section.
 104 sed $E -e "/$header/,/^[[:space:]]+===.*Summary ===/!d" \
 105   -e '/^[A-Z]+:/!d' \
 106   -e '/^(WARNING|ERROR):/d' \
 107   -e 's/\r$//' \
 108   -e 's/^/N:/' \
 109   $NFILE |
 110   sort -s -t : -k 3b - \
 111   >$TMPDIR/n$$-$NBASE
 112
 113 # Merge the two files, then compare adjacent lines.
 114 # Comparison is complicated by tests that may be run multiple times.
 115 # If that case, we assume that the order is the same in both files.
 116 cat <<EOF >compare-$$.awk
 117 BEGIN {
 118     FS = ":"
 119     queue1 = 1; queueN = 0; status[queue1] = ""; name[queue1] = ""
 120     verbose = verbose + 0       # Make sure it's defined.
 121 }
 122
 123 # FIFO circular queue
 124 function push(st, nm) {
 125     queueN += 1; status[queueN] = st; name[queueN] = nm
 126 }
 127 function peek() {
 128     result = 0
 129     if (queueN >= queue1) result = queue1
 130     return result
 131 }
 132 function drop() {
 133     queue1 += 1
 134     if (queue1 > queueN) { queue1 = 1; queueN = 0; }
 135 }
 136
 137 function compare(st, nm) {
 138     old = peek()
 139     if (old == 0) {
 140         # This new test wasn't run last time.
 141         if(st == "FAIL" || st == "UNRESOLVED" || verbose >= 2) {
 142             # New test fails or we want all changes
 143             printf("NA->%s:%s\n", st, nm)
 144         }
 145     }
 146     else {
 147         # Compare this new test to the first queued old one.
 148         if (verbose >= 4) {
 149             printf("Comparing two lines:\n O:%s:%s\n N:%s:%s\n",
 150              status[old], name[old], st, nm)
 151         }
 152         if (name[old] != nm) {
 153             # The old test wasn't run this time and
 154             # the new test wasn't run last time.
 155             if (verbose >= 2) {
 156                 printf("%s->NA:%s\n", status[old], name[old])
 157                 if (nm != "") printf("NA->%s:%s\n", st, nm)
 158             }
 159             drop()
 160         }
 161         else {
 162             notable = 0
 163             if (status[old] == st) {
 164                 # Status of this test has not changed.
 165                 if (verbose >= 3) printf("%s:%s\n", st, nm)
 166             }
 167             else if(status[old] == "PASS" && st == "XFAIL") {
 168                 if (verbose >= 1) notable = 1
 169             }
 170             else if(status[old] == "PASS" || st == "FAIL") {
 171                 # Test did pass but doesn't now
 172                 # or didn't fail but does now.
 173                 notable = 1
 174             }
 175             else if(st == "PASS") {
 176                 # Test didn't pass but does now.
 177                 if (verbose >= 1) notable = 1
 178             }
 179             else if(verbose >= 2) {
 180                 # Miscellaneous status change.
 181                 notable = 1
 182             }
 183             if (notable > 0) printf("%s->%s:%s\n", status[old], st, nm)
 184             drop()
 185         }
 186     }
 187 }
 188
 189 /^O:/ {
 190     while (old = peek()) {
 191         if (name[old] == \$3) break;
 192         # The queued test is no longer run.
 193         compare("", "");
 194     }
 195     # Save this test for later comparison.
 196     push(\$2, \$3)
 197 }
 198
 199 /^N:/ {
 200     compare(\$2, \$3)
 201 }
 202
 203 END {
 204     while (old = peek()) compare("", "")
 205 }
 206 EOF
 207 sort -m -s -t : -k 3b $TMPDIR/o$$-$OBASE $TMPDIR/n$$-$NBASE |
 208  awk -v verbose=$verbose -f compare-$$.awk /dev/stdin
 209
 210 # Delete the temporary files.
 211 rm -f compare-$$.awk $TMPDIR/o$$-$OBASE $TMPDIR/n$$-$NBASE
 212
 213 exit 0