xapian-core/tests/harness/testsuite.h

   1 /** @file testsuite.h
   2  * @brief a generic test suite engine
   3  */
   4 /* Copyright 1999,2000,2001 BrightStation PLC
   5  * Copyright 2002,2003,2005,2006,2007,2008,2009,2013,2015,2016 Olly Betts
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  20  * USA
  21  */
  22
  23 #ifndef OM_HGUARD_TESTSUITE_H
  24 #define OM_HGUARD_TESTSUITE_H
  25
  26 #include "noreturn.h"
  27
  28 #ifndef XAPIAN_UNITTEST
  29 # include "output.h"
  30 # define UNITTEST_CHECK_EXCEPTION
  31 #endif
  32
  33 #include "stringutils.h" // For STRINGIZE().
  34
  35 #include <iomanip>
  36 #include <map>
  37 #include <sstream>
  38 #include <string>
  39 #include <vector>
  40
  41 #include <cfloat> // For DBL_DIG.
  42
  43 /** Class which is thrown when a test case fails.
  44  */
  45 class TestFail { };
  46
  47 /** Class which is thrown when a test case is to be skipped.
  48  *
  49  *  This happens when something can't be tested for some reason, but
  50  *  that reason isn't grounds for causing the test to fail.
  51  */
  52 class TestSkip { };
  53
  54 /** Macro used to build a TestFail object and throw it.
  55  */
  56 // Don't bracket a, because it may have <<'s in it
  57 #define FAIL_TEST(a) do { if (verbose) { tout << a << '\n'; } \
  58                           throw TestFail(); } while (0)
  59
  60 /** Macro used to build a TestSkip object and throw it.
  61  */
  62 // Don't bracket a, because it may have <<'s in it
  63 #define SKIP_TEST(a) do { if (verbose) { tout << a << '\n'; } \
  64                           throw TestSkip(); } while (0)
  65
  66 /// Type for a test function.
  67 typedef bool (*test_func)();
  68
  69 /// Structure holding a description of a test.
  70 struct test_desc {
  71     /// The name of the test.
  72     const char *name;
  73
  74     /// The function to run to perform the test.
  75     test_func run;
  76 };
  77
  78 /// The global verbose flag.
  79 //
  80 //  If verbose is non-zero, then the test harness will display diagnostic output
  81 //  for tests which fail or skip.  If it is > 1, then the diagnostic output will
  82 //  also be displayed for tests which pass.  Individual tests may use this flag
  83 //  to avoid needless generation of diagnostic output in cases when it's
  84 //  expensive.
  85 extern int verbose;
  86
  87 /// The exception type we were expecting in TEST_EXCEPTION.
  88 //  Used to detect if such an exception was mishandled by the
  89 //  compiler/runtime.
  90 extern const char * expected_exception;
  91
  92 /** The output stream.  Data written to this stream will only appear
  93  *  when a test fails.
  94  */
  95 extern std::ostringstream tout;
  96
  97 /// The test driver.  This class takes care of running the tests.
  98 class test_driver {
  99         /// Write out anything in tout and clear it.
 100         void write_and_clear_tout();
 101
 102     public:
 103         /** A structure used to report the summary of tests passed
 104          *  and failed.
 105          */
 106         struct result {
 107             /// The number of tests which succeeded.
 108             unsigned int succeeded;
 109
 110             /// The number of tests which failed.
 111             unsigned int failed;
 112
 113             /// The number of tests which were skipped
 114             unsigned int skipped;
 115
 116             result() : succeeded(0), failed(0), skipped(0) { }
 117
 118             result & operator+=(const result & o) {
 119                 succeeded += o.succeeded;
 120                 failed += o.failed;
 121                 skipped += o.skipped;
 122                 return *this;
 123             }
 124
 125             void reset() {
 126                 succeeded = 0;
 127                 failed = 0;
 128                 skipped = 0;
 129             }
 130         };
 131
 132         /** Add a test-specific command line option.
 133          *
 134          *  The recognised option will be described as:
 135          *
 136          *   -<s> <l>
 137          *
 138          *  And any value set will be put into arg.
 139          */
 140         static void add_command_line_option(const std::string &l, char s,
 141                                             std::string * arg);
 142
 143         /** Parse the command line arguments.
 144          *
 145          *  @param  argc        The argument count passed into ::main()
 146          *  @param  argv        The argument list passed into ::main()
 147          */
 148         static void parse_command_line(int argc, char **argv);
 149
 150         XAPIAN_NORETURN(static void usage());
 151
 152         static int run(const test_desc *tests);
 153
 154         /** The constructor, which sets up the test driver.
 155          *
 156          *  @param tests The zero-terminated array of tests to run.
 157          */
 158         test_driver(const test_desc *tests_);
 159
 160         /** Run all the tests supplied and return the results
 161          */
 162         result run_tests();
 163
 164         /** Run the tests in the list and return the results
 165          */
 166         result run_tests(std::vector<std::string>::const_iterator b,
 167                          std::vector<std::string>::const_iterator e);
 168
 169         /** Read srcdir from environment and if not present, make a valiant
 170          *  attempt to guess a value
 171          */
 172         static std::string get_srcdir();
 173
 174         // Running subtotal for current backend.
 175         static result subtotal;
 176
 177         // Running total for the whole test run.
 178         static result total;
 179
 180         /// Print summary of tests passed, failed, and skipped.
 181         static void report(const test_driver::result &r, const std::string &desc);
 182
 183     private:
 184         /** Prevent copying */
 185         test_driver(const test_driver &);
 186         test_driver & operator = (const test_driver &);
 187
 188         typedef enum { PASS = 1, FAIL = 0, SKIP = -1 } test_result;
 189
 190         static std::map<int, std::string *> short_opts;
 191
 192         static std::string opt_help;
 193
 194         static std::vector<std::string> test_names;
 195
 196         /** Runs the test function and returns its result.  It will
 197          *  also trap exceptions and some memory leaks and force a
 198          *  failure in those cases.
 199          *
 200          *  @param test A description of the test to run.
 201          */
 202         test_result runtest(const test_desc *test);
 203
 204         /** The implementation used by run_tests.
 205          *  it runs test(s) (with runtest()), prints out messages for
 206          *  the user, and tracks the successes and failures.
 207          *
 208          *  @param b, e  If b != e, a vector of the test(s) to run.
 209          *               If b == e, all tests will be run.
 210          */
 211         result do_run_tests(std::vector<std::string>::const_iterator b,
 212                             std::vector<std::string>::const_iterator e);
 213
 214         // abort tests at the first failure
 215         static bool abort_on_error;
 216
 217         // the default stream to output to
 218         std::ostream out;
 219
 220         // the list of tests to run.
 221         const test_desc *tests;
 222
 223         // how many test runs we've done - no summary if just one run
 224         static int runs;
 225
 226         // program name
 227         static std::string argv0;
 228
 229         // strings to use for colouring - empty if output isn't a tty
 230         static std::string col_red, col_green, col_yellow, col_reset;
 231
 232         // use \r to not advance a line when a test passes (this only
 233         // really makes sense if the output is a tty)
 234         static bool use_cr;
 235 };
 236
 237 /// Display the location at which a testcase occurred, with an explanation.
 238 #define TESTCASE_LOCN(a) __FILE__ ":" STRINGIZE(__LINE__) ": " STRINGIZE(a)
 239
 240 /** Test a condition, and display the test with an extra explanation if
 241  *  the condition fails.
 242  *  NB: wrapped in do { ... } while (0) so a trailing ';' works correctly.
 243  */
 244 #define TEST_AND_EXPLAIN(a, b) do {\
 245         bool test_and_explain_fail_ = !(a);\
 246         UNITTEST_CHECK_EXCEPTION\
 247         if (test_and_explain_fail_) FAIL_TEST(TESTCASE_LOCN(a) << std::endl << b << std::endl);\
 248     } while (0)
 249
 250 /// Test a condition, without an additional explanation for failure.
 251 #define TEST(a) TEST_AND_EXPLAIN(a, "")
 252
 253 /// Test for equality of two things.
 254 #define TEST_EQUAL(a, b) TEST_AND_EXPLAIN(((a) == (b)), \
 255         "Expected '" STRINGIZE(a) "' and '" STRINGIZE(b) "' to be equal:" \
 256         " were " << (a) << " and " << (b))
 257
 258 /** Test for equality of two strings.
 259  *
 260  *  If they aren't equal, show each on a separate line so the difference can
 261  *  be seen clearly.
 262  */
 263 #define TEST_STRINGS_EQUAL(a, b) TEST_AND_EXPLAIN(((a) == (b)), \
 264         "Expected " STRINGIZE(a) " and " STRINGIZE(b) " to be equal, were:\n\"" \
 265         << (a) << "\"\n\"" << (b) << '"')
 266
 267 /// Helper function for TEST_EQUAL_DOUBLE macro.
 268 extern bool TEST_EQUAL_DOUBLE_(double a, double b);
 269
 270 /// Test two doubles for near equality.
 271 #define TEST_EQUAL_DOUBLE(a, b) TEST_AND_EXPLAIN(TEST_EQUAL_DOUBLE_((a), (b)), \
 272         "Expected '" STRINGIZE(a) "' and '" STRINGIZE(b) "' to be (nearly) equal:" \
 273         " were " << setprecision(DBL_DIG) << (a) << " and " << (b) << ")" << setprecision(6))
 274
 275 /// Test two doubles for non-near-equality.
 276 #define TEST_NOT_EQUAL_DOUBLE(a, b) TEST_AND_EXPLAIN(!TEST_EQUAL_DOUBLE_((a), (b)), \
 277         "Expected '" STRINGIZE(a) "' and '" STRINGIZE(b) "' not to be (nearly) equal:" \
 278         " were " << setprecision(DBL_DIG) << (a) << " and " << (b) << ")" << setprecision(6))
 279
 280 /// Test for non-equality of two things.
 281 #define TEST_NOT_EQUAL(a, b) TEST_AND_EXPLAIN(((a) != (b)), \
 282         "Expected '" STRINGIZE(a) "' and '" STRINGIZE(b) "' not to be equal:" \
 283         " were " << (a) << " and " << (b))
 284
 285 #define DEFINE_TESTCASE(S,COND) bool test_##S()
 286
 287 // Newer test macros:
 288 #include "testmacros.h"
 289
 290 #endif // OM_HGUARD_TESTSUITE_H