testing/run_regressions.sh

   1 #!/bin/bash
   2 #
   3 # Run the regression tests and display differences
   4 #
   5 DBNAME=regression
   6 PGPOOL=inactive
   7 HOSTNAME=regression
   8
   9 # We need to run the regression tests in the timezone they were written for.
  10 export PGTZ=Pacific/Auckland
  11 export TZ=Pacific/Auckland
  12
  13 ALLSUITES="regression-suite binding carddav scheduling timezone"
  14
  15 . ./regression.conf
  16
  17 [ -z "${DSN}" ] && DSN="${DBNAME}"
  18 [ -n "${HOSTNAME}" ] && WEBHOST="--webhost ${HOSTNAME}"
  19 [ -n "${ALTHOST}"  ] && ALTHOST="--althost ${ALTHOST}"
  20
  21
  22 SUITE=${1:-"regression-suite"}
  23 ACCEPT_ALL=${2:-""}
  24 [ -z "${UNTIL}" ] && UNTIL=99999
  25 [ -z "${SUITE}" ] && SUITE="regression-suite"
  26
  27
  28
  29 check_result() {
  30   TEST="$1"
  31   SKIPDIFF="$2"
  32   if [ ! -f "${REGRESSION}/${TEST}.result" ] ; then
  33     touch "${REGRESSION}/${TEST}.result"
  34   fi
  35   diff --text -u "${REGRESSION}/${TEST}.result" "${RESULTS}/${TEST}" >"${REGRESSION}/diffs/${TEST}"
  36
  37   if [ -s "${REGRESSION}/diffs/${TEST}" ] ; then
  38     if [ -z "$SKIPDIFF" ]; then
  39        echo "======================================="
  40        echo "Displaying diff for test ${TEST}"
  41        echo "======================================="
  42        cat "${REGRESSION}/diffs/${TEST}"
  43        echo "======================================="
  44     fi
  45     if [ "${ACCEPT_ALL}" = "" ] ; then
  46       read -p "[${TEST}] Accept new result [e/s/r/v/f/m/x/y/w/N]? " ACCEPT
  47     else
  48       ACCEPT=${ACCEPT_ALL}
  49     fi
  50     if [ "${ACCEPT}" = "y" ] ; then
  51       cp "${RESULTS}/${TEST}" "${REGRESSION}/${TEST}.result"
  52     elif [ "${ACCEPT}" = "x" ]; then
  53       echo "./dav_test --dsn '${DSN}' ${WEBHOST} ${ALTHOST} --suite '${SUITE}' --case '${TEST}' --debug"
  54       exit
  55     elif [ "${ACCEPT}" = "v" ]; then
  56       echo "Showing test $REGRESSION/${TEST}.test"
  57       cat "$REGRESSION/${TEST}.test"
  58       return 2
  59     elif [ "${ACCEPT}" = "s" ]; then
  60       echo "Displaying side-by-side diff of ${TEST} results"
  61       sdiff --text "${REGRESSION}/${TEST}.result" "${RESULTS}/${TEST}" | less -R
  62       return 3
  63     elif [ "${ACCEPT}" = "w" ]; then
  64       echo "Displaying colourized diff of ${TEST} results"
  65       wdiff -n -w $'\033[30;41m' -x $'\033[0m' -y $'\033[30;42m' -z $'\033[0m' "${REGRESSION}/${TEST}.result" "${RESULTS}/${TEST}" | less -R
  66       return 3
  67     elif [ "${ACCEPT}" = "m" ]; then
  68       echo "Displaying side-by-side 'meld' of ${TEST} results"
  69       meld "${REGRESSION}/${TEST}.result" "${RESULTS}/${TEST}"
  70       return 3
  71     elif [ "${ACCEPT}" = "f" ]; then
  72       echo "Showing full details of ${TEST}"
  73       cat "${REGRESSION}/${TEST}.test"
  74       echo "Showing full result of ${TEST}"
  75       cat "${RESULTS}/${TEST}"
  76       return 3
  77     elif [ "${ACCEPT}" = "e" ]; then
  78       echo "Editing test $REGRESSION/${TEST}.test"
  79       vi "$REGRESSION/${TEST}.test"
  80       return 3
  81     elif [ "${ACCEPT}" = "r" ]; then
  82       echo "Rerunning test ${TEST}"
  83       return 1
  84     fi
  85   else
  86     echo "Test ${TEST} passed OK!"
  87   fi
  88   return 0
  89 }
  90
  91 drop_database() {
  92   dropdb $1
  93   if psql -ltA | cut -f1 -d'|' | grep "^$1$" >/dev/null ; then
  94     # Restart PGPool to ensure we can drop and recreate the database
  95     # FIXME: We should really drop everything *from* the database and create it
  96     # from that, so we don't need to do this.
  97     [ "${PGPOOL}" = "inactive" ] || sudo /etc/init.d/pgpool restart
  98     dropdb $1
  99     if psql -ltA | cut -f1 -d'|' | grep "^$1$" >/dev/null ; then
 100       echo "Failed to drop $1 database"
 101       exit 1
 102     fi
 103   fi
 104 }
 105
 106
 107 restore_database() {
 108   drop_database ${DBNAME}
 109
 110   TEST="Restore-Database"
 111   createdb --owner davical_dba --encoding UTF8 ${DBNAME} >"${RESULTS}/${TEST}" 2>&1
 112   psql ${DBNAME} -q -f "${REGRESSION}/initial.dbdump" >>"${RESULTS}/${TEST}" 2>&1
 113   check_result "${TEST}"
 114 }
 115
 116
 117 dump_database() {
 118   TEST="Dump-Database"
 119   pg_dump -Fp ${DBNAME} > "${REGRESSION}/initial.dbdump" 2>&1
 120 }
 121
 122
 123 initialise_regression() {
 124   drop_database ${DBNAME}
 125
 126   TEST="Create-Database"
 127   ../dba/create-database.sh ${DBNAME} 'nimda' >"${RESULTS}/${TEST}" 2>&1
 128   check_result "${TEST}"
 129
 130   TEST="Upgrade-Database"
 131   ../dba/update-davical-database --dbname=${DBNAME} --nopatch --appuser davical_app --owner davical_dba >"${RESULTS}/${TEST}" 2>&1
 132   check_result "${TEST}"
 133
 134   if [ -f "${REGRESSION}/sample-data.sql" ]; then
 135     TEST="Load-Sample-Data"
 136     psql -q -f "${REGRESSION}/sample-data.sql" "${DBNAME}" >"${RESULTS}/${TEST}" 2>&1
 137     check_result "${TEST}"
 138   fi
 139
 140   TEST="Really-Upgrade-Database"
 141   ../dba/update-davical-database --dbname=${DBNAME} --appuser davical_app --owner davical_dba >"${RESULTS}/${TEST}" 2>&1
 142   check_result "${TEST}"
 143
 144 }
 145
 146
 147 run_regression_suite() {
 148   RESULTS="${REGRESSION}/results"
 149   mkdir -p "${RESULTS}"
 150   mkdir -p "${REGRESSION}/diffs"
 151
 152   if [ -f "${REGRESSION}/initial.dbdump" ]; then
 153     restore_database
 154   else
 155     initialise_regression
 156   fi
 157
 158   for T in ${REGRESSION}/*.test ; do
 159     [ -f "${T}" ] || break
 160     TEST="`basename ${T} .test`"
 161     TESTNUM="`echo ${TEST} | cut -f1 -d'-'`"
 162     TESTNUM="${TEST/-*}"
 163     if [ "${TESTNUM}" -gt "${UNTIL}" ] ; then
 164       break;
 165     fi
 166
 167     RESULT=999
 168     while [ "${RESULT}" -gt 0 ]; do
 169       ./dav_test --dsn "${DSN}" ${WEBHOST} ${ALTHOST} --suite "${SUITE}" --case "${TEST}" | ./normalise_result > "${RESULTS}/${TEST}"
 170       # Fix Vim syntax highlighting by putting an esac here.  Silly, huh?
 171
 172       RESULT=999
 173       SKIPDIFF=""
 174       while [ "${RESULT}" -gt 1 ]; do
 175         check_result "${TEST}" "$SKIPDIFF"
 176         RESULT=$?
 177         SKIPDIFF=""
 178         [ "${RESULT}" -gt 2 ] && SKIPDIFF=1
 179       done
 180
 181     done
 182
 183     TCOUNT="$(( ${TCOUNT} + 1 ))"
 184   done
 185 }
 186
 187
 188
 189 TSTART="`date +%s`"
 190 TCOUNT=0
 191
 192 if [ "${SUITE}" = "all" ]; then
 193   for SUITE in ${ALLSUITES} ; do
 194     REGRESSION="tests/${SUITE}"
 195     if [ "${SUITE}" != "regression-suite" ]; then
 196       dump_database
 197     fi
 198     run_regression_suite "${SUITE}"
 199   done
 200 else
 201   REGRESSION="tests/${SUITE}"
 202   run_regression_suite "${SUITE}"
 203 fi
 204 TFINISH="`date +%s`"
 205
 206 echo "Regression test run took $(( ${TFINISH} - ${TSTART} )) seconds for ${TCOUNT} tests."