testing/run_regressions.sh

   1 #!/bin/bash
   2 #
   3 # Run the regression tests and display differences
   4 #
   5 DBNAME=regression
   6 PGPOOL=inactive
   7 HOSTNAME=regression
   8
   9 # We need to run the regression tests in the timezone they were written for.
  10 export PGTZ=Pacific/Auckland
  11
  12 ALLSUITES="regression-suite binding carddav scheduling timezone"
  13
  14 . ./regression.conf
  15
  16 [ -z "${DSN}" ] && DSN="${DBNAME}"
  17 [ -n "${HOSTNAME}" ] && WEBHOST="--webhost ${HOSTNAME}"
  18 [ -n "${ALTHOST}"  ] && ALTHOST="--althost ${ALTHOST}"
  19
  20
  21 SUITE=${1:-"regression-suite"}
  22 ACCEPT_ALL=${2:-""}
  23 [ -z "${UNTIL}" ] && UNTIL=99999
  24 [ -z "${SUITE}" ] && SUITE="regression-suite"
  25
  26
  27
  28 check_result() {
  29   TEST="$1"
  30   SKIPDIFF="$2"
  31   if [ ! -f "${REGRESSION}/${TEST}.result" ] ; then
  32     touch "${REGRESSION}/${TEST}.result"
  33   fi
  34   diff --text -u "${REGRESSION}/${TEST}.result" "${RESULTS}/${TEST}" >"${REGRESSION}/diffs/${TEST}"
  35
  36   if [ -s "${REGRESSION}/diffs/${TEST}" ] ; then
  37     if [ -z "$SKIPDIFF" ]; then
  38        echo "======================================="
  39        echo "Displaying diff for test ${TEST}"
  40        echo "======================================="
  41        cat "${REGRESSION}/diffs/${TEST}"
  42        echo "======================================="
  43     fi
  44     if [ "${ACCEPT_ALL}" = "" ] ; then
  45       read -p "[${TEST}] Accept new result [e/s/r/v/f/m/x/y/w/N]? " ACCEPT
  46     else
  47       ACCEPT=${ACCEPT_ALL}
  48     fi
  49     if [ "${ACCEPT}" = "y" ] ; then
  50       cp "${RESULTS}/${TEST}" "${REGRESSION}/${TEST}.result"
  51     elif [ "${ACCEPT}" = "x" ]; then
  52       echo "./dav_test --dsn '${DSN}' ${WEBHOST} ${ALTHOST} --suite '${SUITE}' --case '${TEST}' --debug"
  53       exit
  54     elif [ "${ACCEPT}" = "v" ]; then
  55       echo "Showing test $REGRESSION/${TEST}.test"
  56       cat "$REGRESSION/${TEST}.test"
  57       return 2
  58     elif [ "${ACCEPT}" = "s" ]; then
  59       echo "Displaying side-by-side diff of ${TEST} results"
  60       sdiff --text "${REGRESSION}/${TEST}.result" "${RESULTS}/${TEST}" | less -R
  61       return 3
  62     elif [ "${ACCEPT}" = "w" ]; then
  63       echo "Displaying colourized diff of ${TEST} results"
  64       wdiff -n -w $'\033[30;41m' -x $'\033[0m' -y $'\033[30;42m' -z $'\033[0m' "${REGRESSION}/${TEST}.result" "${RESULTS}/${TEST}" | less -R
  65       return 3
  66     elif [ "${ACCEPT}" = "m" ]; then
  67       echo "Displaying side-by-side 'meld' of ${TEST} results"
  68       meld "${REGRESSION}/${TEST}.result" "${RESULTS}/${TEST}"
  69       return 3
  70     elif [ "${ACCEPT}" = "f" ]; then
  71       echo "Showing full details of ${TEST}"
  72       cat "${REGRESSION}/${TEST}.test"
  73       echo "Showing full result of ${TEST}"
  74       cat "${RESULTS}/${TEST}"
  75       return 3
  76     elif [ "${ACCEPT}" = "e" ]; then
  77       echo "Editing test $REGRESSION/${TEST}.test"
  78       vi "$REGRESSION/${TEST}.test"
  79       return 3
  80     elif [ "${ACCEPT}" = "r" ]; then
  81       echo "Rerunning test ${TEST}"
  82       return 1
  83     fi
  84   else
  85     echo "Test ${TEST} passed OK!"
  86   fi
  87   return 0
  88 }
  89
  90 drop_database() {
  91   dropdb $1
  92   if psql -ltA | cut -f1 -d'|' | grep "^$1$" >/dev/null ; then
  93     # Restart PGPool to ensure we can drop and recreate the database
  94     # FIXME: We should really drop everything *from* the database and create it
  95     # from that, so we don't need to do this.
  96     [ "${PGPOOL}" = "inactive" ] || sudo /etc/init.d/pgpool restart
  97     dropdb $1
  98     if psql -ltA | cut -f1 -d'|' | grep "^$1$" >/dev/null ; then
  99       echo "Failed to drop $1 database"
 100       exit 1
 101     fi
 102   fi
 103 }
 104
 105
 106 restore_database() {
 107   drop_database ${DBNAME}
 108
 109   TEST="Restore-Database"
 110   createdb --owner davical_dba --encoding UTF8 ${DBNAME} >"${RESULTS}/${TEST}" 2>&1
 111   psql ${DBNAME} -q -f "${REGRESSION}/initial.dbdump" >>"${RESULTS}/${TEST}" 2>&1
 112   check_result "${TEST}"
 113 }
 114
 115
 116 dump_database() {
 117   TEST="Dump-Database"
 118   pg_dump -Fp ${DBNAME} > "${REGRESSION}/initial.dbdump" 2>&1
 119 }
 120
 121
 122 initialise_regression() {
 123   drop_database ${DBNAME}
 124
 125   TEST="Create-Database"
 126   ../dba/create-database.sh ${DBNAME} 'nimda' >"${RESULTS}/${TEST}" 2>&1
 127   check_result "${TEST}"
 128
 129   TEST="Upgrade-Database"
 130   ../dba/update-davical-database --dbname=${DBNAME} --nopatch --appuser davical_app --owner davical_dba >"${RESULTS}/${TEST}" 2>&1
 131   check_result "${TEST}"
 132
 133   if [ -f "${REGRESSION}/sample-data.sql" ]; then
 134     TEST="Load-Sample-Data"
 135     psql -q -f "${REGRESSION}/sample-data.sql" "${DBNAME}" >"${RESULTS}/${TEST}" 2>&1
 136     check_result "${TEST}"
 137   fi
 138
 139   TEST="Really-Upgrade-Database"
 140   ../dba/update-davical-database --dbname=${DBNAME} --appuser davical_app --owner davical_dba >"${RESULTS}/${TEST}" 2>&1
 141   check_result "${TEST}"
 142
 143 }
 144
 145
 146 run_regression_suite() {
 147   RESULTS="${REGRESSION}/results"
 148   mkdir -p "${RESULTS}"
 149   mkdir -p "${REGRESSION}/diffs"
 150
 151   if [ -f "${REGRESSION}/initial.dbdump" ]; then
 152     restore_database
 153   else
 154     initialise_regression
 155   fi
 156
 157   for T in ${REGRESSION}/*.test ; do
 158     [ -f "${T}" ] || break
 159     TEST="`basename ${T} .test`"
 160     TESTNUM="`echo ${TEST} | cut -f1 -d'-'`"
 161     TESTNUM="${TEST/-*}"
 162     if [ "${TESTNUM}" -gt "${UNTIL}" ] ; then
 163       break;
 164     fi
 165
 166     RESULT=999
 167     while [ "${RESULT}" -gt 0 ]; do
 168       ./dav_test --dsn "${DSN}" ${WEBHOST} ${ALTHOST} --suite "${SUITE}" --case "${TEST}" | ./normalise_result > "${RESULTS}/${TEST}"
 169       # Fix Vim syntax highlighting by putting an esac here.  Silly, huh?
 170
 171       RESULT=999
 172       SKIPDIFF=""
 173       while [ "${RESULT}" -gt 1 ]; do
 174         check_result "${TEST}" "$SKIPDIFF"
 175         RESULT=$?
 176         SKIPDIFF=""
 177         [ "${RESULT}" -gt 2 ] && SKIPDIFF=1
 178       done
 179
 180     done
 181
 182     TCOUNT="$(( ${TCOUNT} + 1 ))"
 183   done
 184 }
 185
 186
 187
 188 TSTART="`date +%s`"
 189 TCOUNT=0
 190
 191 if [ "${SUITE}" = "all" ]; then
 192   for SUITE in ${ALLSUITES} ; do
 193     REGRESSION="tests/${SUITE}"
 194     if [ "${SUITE}" != "regression-suite" ]; then
 195       dump_database
 196     fi
 197     run_regression_suite "${SUITE}"
 198   done
 199 else
 200   REGRESSION="tests/${SUITE}"
 201   run_regression_suite "${SUITE}"
 202 fi
 203 TFINISH="`date +%s`"
 204
 205 echo "Regression test run took $(( ${TFINISH} - ${TSTART} )) seconds for ${TCOUNT} tests."