xapian-core/tests/harness/testsuite.cc

   1 /* testsuite.cc: a test suite engine
   2  *
   3  * Copyright 1999,2000,2001 BrightStation PLC
   4  * Copyright 2002 Ananova Ltd
   5  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2012,2013,2015,2016,2017 Olly Betts
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  20  * USA
  21  */
  22
  23 #include <config.h>
  24
  25 #include "testsuite.h"
  26
  27 #ifndef NO_LIBXAPIAN
  28 # include "backendmanager.h"
  29 #endif
  30 #include "fdtracker.h"
  31 #include "testrunner.h"
  32 #include "safeerrno.h"
  33 #include "safeunistd.h"
  34
  35 #ifdef HAVE_VALGRIND
  36 # include <valgrind/memcheck.h>
  37 # include <sys/types.h>
  38 # include "safefcntl.h"
  39 #endif
  40
  41 #include <algorithm>
  42 #include <iomanip>
  43 #include <iostream>
  44 #include <set>
  45
  46 #include <cfloat> // For DBL_DIG.
  47 #include <cmath> // For ceil, fabs, log10.
  48 #include <cstdio>
  49 #include <cstdlib>
  50 #include <cstring>
  51
  52 #include "gnu_getopt.h"
  53
  54 #include <setjmp.h>
  55 #include <signal.h>
  56
  57 #include <exception>
  58 #ifdef USE_RTTI
  59 # include <typeinfo>
  60 # ifdef HAVE_CXXABI_H
  61 #  include <cxxabi.h>
  62 # endif
  63 #endif
  64
  65 #ifndef NO_LIBXAPIAN
  66 # include <xapian/error.h>
  67 #endif
  68 #include "filetests.h"
  69 #include "stringutils.h"
  70
  71 using namespace std;
  72
  73 /// The global verbose flag.
  74 int verbose;
  75
  76 #ifdef HAVE_VALGRIND
  77 static int vg_log_fd = -1;
  78 #endif
  79
  80 #ifdef HAVE_SIGSETJMP
  81 # define SIGSETJMP(ENV, SAVESIGS) sigsetjmp(ENV, SAVESIGS)
  82 # define SIGLONGJMP(ENV, VAL) siglongjmp(ENV, VAL)
  83 # define SIGJMP_BUF sigjmp_buf
  84 #else
  85 # define SIGSETJMP(ENV, SAVESIGS) setjmp(ENV)
  86 # define SIGLONGJMP(ENV, VAL) longjmp(ENV, VAL)
  87 # define SIGJMP_BUF jmp_buf
  88 #endif
  89
  90 /// The exception type we were expecting in TEST_EXCEPTION
  91 //  We use this to attempt to diagnose when the code fails to catch an
  92 //  exception when it should (due to a compiler or runtime fault in
  93 //  GCC 2.95 it seems)
  94 const char * expected_exception = NULL;
  95
  96 /// The debug printing stream
  97 std::ostringstream tout;
  98
  99 int test_driver::runs = 0;
 100 test_driver::result test_driver::subtotal;
 101 test_driver::result test_driver::total;
 102 string test_driver::argv0;
 103 string test_driver::opt_help;
 104 map<int, string *> test_driver::short_opts;
 105 vector<string> test_driver::test_names;
 106 bool test_driver::abort_on_error = false;
 107 string test_driver::col_red, test_driver::col_green;
 108 string test_driver::col_yellow, test_driver::col_reset;
 109 bool test_driver::use_cr = false;
 110
 111 void
 112 test_driver::write_and_clear_tout()
 113 {
 114     const string & s = tout.str();
 115     if (!s.empty()) {
 116         out << '\n' << s;
 117         tout.str(string());
 118     }
 119 }
 120
 121 string
 122 test_driver::get_srcdir()
 123 {
 124     char *p = getenv("srcdir");
 125     if (p != NULL) return string(p);
 126
 127     // Default srcdir to the pathname of argv[0].
 128     string srcdir(argv0);
 129     string::size_type i = srcdir.find_last_of(DIR_SEPS);
 130     string srcfile;
 131     if (i != string::npos) {
 132         srcfile.assign(srcdir, i + 1, string::npos);
 133         srcdir.erase(i);
 134         // libtool may put the real executable in .libs.
 135         i = srcdir.find_last_of(DIR_SEPS);
 136         if (srcdir.substr(i + 1) == ".libs") {
 137             srcdir.erase(i);
 138             // And it may have an "lt-" prefix.
 139             if (startswith(srcfile, "lt-")) srcfile.erase(0, 3);
 140         }
 141     } else {
 142         // No path of argv[0], so default srcdir to the current directory.
 143         // This may not work if libtool is involved as the true executable is
 144         // sometimes in ".libs".
 145         srcfile = srcdir;
 146         srcdir = ".";
 147     }
 148
 149     // Remove any trailing ".exe" suffix, since some platforms add this.
 150     if (endswith(srcfile, ".exe")) srcfile.resize(srcfile.size() - 4);
 151
 152     // Sanity check.
 153     if (!file_exists(srcdir + '/' + srcfile + ".cc")) {
 154         cout << argv0
 155              << ": srcdir is not in the environment and I can't guess it!\n"
 156                 "Run test programs using the runtest script - see HACKING for details"
 157              << endl;
 158         exit(1);
 159     }
 160     return srcdir;
 161 }
 162
 163 test_driver::test_driver(const test_desc *tests_)
 164         : out(cout.rdbuf()), tests(tests_)
 165 {
 166     tout << boolalpha;
 167 }
 168
 169 static SIGJMP_BUF jb;
 170 static int signum = 0;
 171 static void * sigaddr = NULL;
 172
 173 // Needs C linkage so we can pass it to sigaction()/signal() without problems.
 174 extern "C" {
 175
 176 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 177 [[noreturn]]
 178 static void handle_sig(int signum_, siginfo_t *si, void *)
 179 {
 180     // Disable all our signal handlers to avoid problems if the signal
 181     // handling code causes a signal.
 182     struct sigaction sa;
 183     sa.sa_handler = SIG_DFL;
 184     sigemptyset(&sa.sa_mask);
 185     sa.sa_flags = 0;
 186     // We set the handlers with SA_RESETHAND, but that will only reset the
 187     // handler for the signal which fired.
 188     if (signum_ != SIGSEGV) sigaction(SIGSEGV, &sa, NULL);
 189     if (signum_ != SIGFPE) sigaction(SIGFPE, &sa, NULL);
 190     if (signum_ != SIGILL) sigaction(SIGILL, &sa, NULL);
 191 # ifdef SIGBUS
 192     if (signum_ != SIGBUS) sigaction(SIGBUS, &sa, NULL);
 193 # endif
 194 # ifdef SIGSTKFLT
 195     if (signum_ != SIGSTKFLT) sigaction(SIGSTKFLT, &sa, NULL);
 196 # endif
 197     signum = signum_;
 198     sigaddr = si->si_addr;
 199     SIGLONGJMP(jb, 1);
 200 }
 201
 202 #else
 203
 204 [[noreturn]]
 205 static void handle_sig(int signum_)
 206 {
 207     // Disable all our signal handlers to avoid problems if the signal
 208     // handling code causes a signal.
 209     signal(SIGSEGV, SIG_DFL);
 210     signal(SIGFPE, SIG_DFL);
 211     signal(SIGILL, SIG_DFL);
 212 #ifdef SIGBUS
 213     signal(SIGBUS, SIG_DFL);
 214 #endif
 215 #ifdef SIGSTKFLT
 216     signal(SIGSTKFLT, SIG_DFL);
 217 #endif
 218     signum = signum_;
 219     SIGLONGJMP(jb, 1);
 220 }
 221 #endif
 222
 223 }
 224
 225 class SignalRedirector {
 226   private:
 227     bool active;
 228   public:
 229     SignalRedirector() : active(false) { }
 230     void activate() {
 231         active = true;
 232         signum = 0;
 233         sigaddr = NULL;
 234         // SA_SIGINFO not universal (e.g. not present on Linux < 2.2 and Hurd).
 235 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 236         struct sigaction sa;
 237         sa.sa_sigaction = handle_sig;
 238         sigemptyset(&sa.sa_mask);
 239         sa.sa_flags = SA_RESETHAND|SA_SIGINFO;
 240         sigaction(SIGSEGV, &sa, NULL);
 241         sigaction(SIGFPE, &sa, NULL);
 242         sigaction(SIGILL, &sa, NULL);
 243 # ifdef SIGBUS
 244         sigaction(SIGBUS, &sa, NULL);
 245 # endif
 246 # ifdef SIGSTKFLT
 247         sigaction(SIGSTKFLT, &sa, NULL);
 248 # endif
 249 #else
 250         signal(SIGSEGV, handle_sig);
 251         signal(SIGFPE, handle_sig);
 252         signal(SIGILL, handle_sig);
 253 # ifdef SIGBUS
 254         signal(SIGBUS, handle_sig);
 255 # endif
 256 # ifdef SIGSTKFLT
 257         signal(SIGSTKFLT, handle_sig);
 258 # endif
 259 #endif
 260     }
 261     ~SignalRedirector() {
 262         if (active) {
 263 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 264             struct sigaction sa;
 265             sa.sa_handler = SIG_DFL;
 266             sigemptyset(&sa.sa_mask);
 267             sa.sa_flags = 0;
 268             sigaction(SIGSEGV, &sa, NULL);
 269             sigaction(SIGFPE, &sa, NULL);
 270             sigaction(SIGILL, &sa, NULL);
 271 # ifdef SIGBUS
 272             sigaction(SIGBUS, &sa, NULL);
 273 # endif
 274 # ifdef SIGSTKFLT
 275             sigaction(SIGSTKFLT, &sa, NULL);
 276 # endif
 277 #else
 278             signal(SIGSEGV, SIG_DFL);
 279             signal(SIGFPE, SIG_DFL);
 280             signal(SIGILL, SIG_DFL);
 281 # ifdef SIGBUS
 282             signal(SIGBUS, SIG_DFL);
 283 # endif
 284 # ifdef SIGSTKFLT
 285             signal(SIGSTKFLT, SIG_DFL);
 286 # endif
 287 #endif
 288         }
 289     }
 290 };
 291
 292 //  A wrapper around the tests to trap exceptions,
 293 //  and avoid having to catch them in every test function.
 294 //  If this test driver is used for anything other than
 295 //  Xapian tests, then this ought to be provided by
 296 //  the client, really.
 297 //  return: test_driver::PASS, test_driver::FAIL, or test_driver::SKIP
 298 test_driver::test_result
 299 test_driver::runtest(const test_desc *test)
 300 {
 301     // This is used to make a note of how many times we've run the test
 302     volatile int runcount = 0;
 303
 304     FDTracker fdtracker;
 305     fdtracker.init();
 306
 307     while (true) {
 308         tout.str(string());
 309         if (SIGSETJMP(jb, 1) == 0) {
 310             SignalRedirector sig; // use object so signal handlers are reset
 311             static bool catch_signals =
 312                 (getenv("XAPIAN_TESTSUITE_SIG_DFL") == NULL);
 313             if (catch_signals) sig.activate();
 314             try {
 315                 expected_exception = NULL;
 316 #ifdef HAVE_VALGRIND
 317                 int vg_errs = 0;
 318                 long vg_leaks = 0, vg_dubious = 0, vg_reachable = 0;
 319                 if (vg_log_fd != -1) {
 320                     VALGRIND_DO_LEAK_CHECK;
 321                     vg_errs = VALGRIND_COUNT_ERRORS;
 322                     long dummy;
 323                     VALGRIND_COUNT_LEAKS(vg_leaks, vg_dubious, vg_reachable, dummy);
 324                     (void)dummy;
 325                     // Skip past any unread log output.
 326                     lseek(vg_log_fd, 0, SEEK_END);
 327                 }
 328 #endif
 329                 if (!test->run()) {
 330                     out << col_red << " FAILED" << col_reset;
 331                     write_and_clear_tout();
 332                     return FAIL;
 333                 }
 334                 if (verbose > 1)
 335                     write_and_clear_tout();
 336 #ifndef NO_LIBXAPIAN
 337                 if (backendmanager)
 338                     backendmanager->clean_up();
 339 #endif
 340 #ifdef HAVE_VALGRIND
 341                 if (vg_log_fd != -1) {
 342                     // We must empty tout before asking valgrind to perform its
 343                     // leak checks, otherwise the buffers holding the output
 344                     // may be identified as a memory leak (especially if >1K of
 345                     // output has been buffered it appears...)
 346                     tout.str(string());
 347 #define REPORT_FAIL_VG(M) do { \
 348     if (verbose) { \
 349         while (true) { \
 350             ssize_t c = read(vg_log_fd, buf, sizeof(buf)); \
 351             if (c == 0 || (c < 0 && errno != EINTR)) break; \
 352             if (c > 0) out << string(buf, c); \
 353         } \
 354     } \
 355     out << " " << col_red << M << col_reset; \
 356 } while (0)
 357                     // Record the current position so we can restore it so
 358                     // REPORT_FAIL_VG() gets the whole output.
 359                     off_t curpos = lseek(vg_log_fd, 0, SEEK_CUR);
 360                     char buf[4096];
 361                     while (true) {
 362                         ssize_t c = read(vg_log_fd, buf, sizeof(buf));
 363                         if (c == 0 || (c < 0 && errno != EINTR)) {
 364                             buf[0] = 0;
 365                             break;
 366                         }
 367                         if (c > 0) {
 368                             // Valgrind output has "==<pid>== \n" between
 369                             // report "records", so skip any lines like that,
 370                             // and also any warnings and continuation lines.
 371                             ssize_t i = 0;
 372                             while (true) {
 373                                 const char * spc;
 374                                 spc = static_cast<const char *>(
 375                                         memchr(buf + i, ' ', c - i));
 376                                 if (!spc) {
 377                                     i = c;
 378                                     break;
 379                                 }
 380                                 i = spc - buf;
 381                                 if (++i >= c) break;
 382                                 if (buf[i] == '\n')
 383                                     continue;
 384                                 if (c - i >= 8 &&
 385                                     (memcmp(buf + i, "Warning:", 8) == 0 ||
 386                                      memcmp(buf + i, "   ", 3) == 0)) {
 387                                     // Skip this line.
 388                                     i += 3;
 389                                     const char * nl;
 390                                     nl = static_cast<const char *>(
 391                                             memchr(buf + i, '\n', c - i));
 392                                     if (!nl) {
 393                                         i = c;
 394                                         break;
 395                                     }
 396                                     i = nl - buf;
 397                                     continue;
 398                                 }
 399                                 break;
 400                             }
 401
 402                             char *start = buf + i;
 403                             c -= i;
 404                             if (c > 128) c = 128;
 405
 406                             {
 407                                 const char *p;
 408                                 p = static_cast<const char*>(
 409                                         memchr(start, '\n', c));
 410                                 if (p != NULL) c = p - start;
 411                             }
 412
 413                             memmove(buf, start, c);
 414                             buf[c] = '\0';
 415                             break;
 416                         }
 417                     }
 418                     lseek(vg_log_fd, curpos, SEEK_SET);
 419
 420                     int vg_errs2 = VALGRIND_COUNT_ERRORS;
 421                     vg_errs = vg_errs2 - vg_errs;
 422                     VALGRIND_DO_LEAK_CHECK;
 423                     long vg_leaks2 = 0, vg_dubious2 = 0, vg_reachable2 = 0;
 424                     long dummy;
 425                     VALGRIND_COUNT_LEAKS(vg_leaks2, vg_dubious2, vg_reachable2,
 426                                          dummy);
 427                     (void)dummy;
 428                     vg_leaks = vg_leaks2 - vg_leaks;
 429                     vg_dubious = vg_dubious2 - vg_dubious;
 430                     vg_reachable = vg_reachable2 - vg_reachable;
 431                     if (vg_errs) {
 432                         string fail_msg(buf);
 433                         if (fail_msg.empty())
 434                             fail_msg = "VALGRIND DETECTED A PROBLEM";
 435                         REPORT_FAIL_VG(fail_msg);
 436                         return FAIL;
 437                     }
 438                     if (vg_leaks > 0) {
 439                         REPORT_FAIL_VG("LEAKED " << vg_leaks << " BYTES");
 440                         return FAIL;
 441                     }
 442                     if (vg_dubious > 0) {
 443                         // If code deliberately holds onto blocks by a pointer
 444                         // not to the start (e.g. languages/utilities.c does)
 445                         // then we need to rerun the test to see if the leak is
 446                         // real...
 447                         if (runcount == 0) {
 448                             out << col_yellow << " PROBABLY LEAKED MEMORY - RETRYING TEST" << col_reset;
 449                             ++runcount;
 450                             // Ensure that any cached memory from fd tracking
 451                             // is allocated before we rerun the test.
 452                             (void)fdtracker.check();
 453                             continue;
 454                         }
 455                         REPORT_FAIL_VG("PROBABLY LEAKED " << vg_dubious << " BYTES");
 456                         return FAIL;
 457                     }
 458                     if (vg_reachable > 0) {
 459                         // C++ STL implementations often "horde" released
 460                         // memory - for GCC 3.4 and newer the runtest script
 461                         // sets GLIBCXX_FORCE_NEW=1 which will disable this
 462                         // behaviour so we avoid this issue, but for older
 463                         // GCC and other compilers this may be an issue.
 464                         //
 465                         // See also:
 466                         // http://valgrind.org/docs/FAQ/#faq.reports
 467                         //
 468                         // For now, just use runcount to rerun the test and see
 469                         // if more is leaked - hopefully this shouldn't give
 470                         // false positives.
 471                         if (runcount == 0) {
 472                             out << col_yellow << " POSSIBLE UNRELEASED MEMORY - RETRYING TEST" << col_reset;
 473                             ++runcount;
 474                             // Ensure that any cached memory from fd tracking
 475                             // is allocated before we rerun the test.
 476                             (void)fdtracker.check();
 477                             continue;
 478                         }
 479                         REPORT_FAIL_VG("FAILED TO RELEASE " << vg_reachable << " BYTES");
 480                         return FAIL;
 481                     }
 482                 }
 483 #endif
 484                 if (!fdtracker.check()) {
 485                     if (runcount == 0) {
 486                         out << col_yellow << " POSSIBLE FDLEAK:" << fdtracker.get_message() << col_reset;
 487                         ++runcount;
 488                         continue;
 489                     }
 490                     out << col_red << " FDLEAK:" << fdtracker.get_message() << col_reset;
 491                     return FAIL;
 492                 }
 493             } catch (const TestFail &) {
 494                 out << col_red << " FAILED" << col_reset;
 495                 write_and_clear_tout();
 496                 return FAIL;
 497             } catch (const TestSkip &) {
 498                 out << col_yellow << " SKIPPED" << col_reset;
 499                 write_and_clear_tout();
 500                 return SKIP;
 501 #ifndef NO_LIBXAPIAN
 502             } catch (const Xapian::Error &err) {
 503                 string errclass = err.get_type();
 504                 if (expected_exception && expected_exception == errclass) {
 505                     out << col_yellow << " C++ FAILED TO CATCH " << errclass << col_reset;
 506                     return SKIP;
 507                 }
 508                 if (errclass == "NetworkError" &&
 509                     err.get_error_string() != NULL &&
 510                     strcmp(err.get_error_string(), strerror(ECHILD)) == 0) {
 511                     // ECHILD suggests we've run out of processes, and that's
 512                     // much more likely to be a system issue than a Xapian bug.
 513                     //
 514                     // We also see apparently spurious ECHILD on Debian
 515                     // buildds sometimes: https://bugs.debian.org/681941
 516                     out << col_yellow << " ECHILD in network code" << col_reset;
 517                     return SKIP;
 518                 }
 519                 out << " " << col_red << err.get_description() << col_reset;
 520                 write_and_clear_tout();
 521                 return FAIL;
 522 #endif
 523             } catch (const string & msg) {
 524                 out << col_red << " EXCEPTION std::string " << msg << col_reset;
 525                 write_and_clear_tout();
 526                 return FAIL;
 527             } catch (const std::exception & e) {
 528                 out << " " << col_red;
 529 #ifndef USE_RTTI
 530                 out << "std::exception";
 531 #else
 532                 const char * name = typeid(e).name();
 533 # ifdef HAVE_CXXABI_H
 534                 // __cxa_demangle() apparently requires GCC >= 3.1.
 535                 // Demangle the name which GCC returns for type_info::name().
 536                 int status;
 537                 char * realname = abi::__cxa_demangle(name, NULL, 0, &status);
 538                 if (realname) {
 539                     out << realname;
 540                     free(realname);
 541                 } else {
 542                     out << name;
 543                 }
 544 # else
 545                 out << name;
 546 # endif
 547 #endif
 548                 out << ": " << e.what() << col_reset;
 549                 write_and_clear_tout();
 550                 return FAIL;
 551             } catch (const char * msg) {
 552                 out << col_red;
 553                 if (msg) {
 554                     out << " EXCEPTION char * " << msg;
 555                 } else {
 556                     out << " EXCEPTION (char*)NULL";
 557                 }
 558                 out << col_reset;
 559                 write_and_clear_tout();
 560                 return FAIL;
 561             } catch (...) {
 562                 out << col_red << " UNKNOWN EXCEPTION" << col_reset;
 563                 write_and_clear_tout();
 564                 return FAIL;
 565             }
 566             return PASS;
 567         }
 568
 569         // Caught a signal.
 570         const char *signame = "SIGNAL";
 571         bool show_addr = true;
 572         switch (signum) {
 573             case SIGSEGV: signame = "SIGSEGV"; break;
 574             case SIGFPE: signame = "SIGFPE"; break;
 575             case SIGILL: signame = "SIGILL"; break;
 576 #ifdef SIGBUS
 577             case SIGBUS: signame = "SIGBUS"; break;
 578 #endif
 579 #ifdef SIGSTKFLT
 580             case SIGSTKFLT:
 581                 signame = "SIGSTKFLT";
 582                 show_addr = false;
 583                 break;
 584 #endif
 585         }
 586         out << " " << col_red << signame;
 587         if (show_addr) {
 588             char buf[40];
 589             sprintf(buf, " at %p", sigaddr);
 590             out << buf;
 591         }
 592         out << col_reset;
 593         write_and_clear_tout();
 594         return FAIL;
 595     }
 596 }
 597
 598 test_driver::result
 599 test_driver::run_tests(vector<string>::const_iterator b,
 600                        vector<string>::const_iterator e)
 601 {
 602     return do_run_tests(b, e);
 603 }
 604
 605 test_driver::result
 606 test_driver::run_tests()
 607 {
 608     const vector<string> blank;
 609     return do_run_tests(blank.begin(), blank.end());
 610 }
 611
 612 test_driver::result
 613 test_driver::do_run_tests(vector<string>::const_iterator b,
 614                           vector<string>::const_iterator e)
 615 {
 616     set<string> m(b, e);
 617     bool check_name = !m.empty();
 618
 619     test_driver::result res;
 620
 621     for (const test_desc *test = tests; test->name; ++test) {
 622         bool do_this_test = !check_name;
 623         if (!do_this_test && m.find(test->name) != m.end())
 624             do_this_test = true;
 625         if (!do_this_test) {
 626             // if this test is "foo123" see if "foo" was listed
 627             // this way "./testprog foo" can run foo1, foo2, etc.
 628             string t = test->name;
 629             string::size_type i;
 630             i = t.find_last_not_of("0123456789") + 1;
 631             if (i != string::npos) {
 632                 t.resize(i);
 633                 if (m.find(t) != m.end()) do_this_test = true;
 634             }
 635         }
 636         if (do_this_test) {
 637             out << "Running test: " << test->name << "...";
 638             out.flush();
 639             test_driver::test_result test_res = runtest(test);
 640 #ifndef NO_LIBXAPIAN
 641             if (backendmanager)
 642                 backendmanager->clean_up();
 643 #endif
 644             switch (test_res) {
 645                 case PASS:
 646                     ++res.succeeded;
 647                     if (verbose || !use_cr) {
 648                         out << col_green << " ok" << col_reset << endl;
 649                     } else {
 650                         out << "\r                                                                               \r";
 651                     }
 652                     break;
 653                 case FAIL:
 654                     ++res.failed;
 655                     out << endl;
 656                     if (abort_on_error) {
 657                         throw "Test failed - aborting further tests";
 658                     }
 659                     break;
 660                 case SKIP:
 661                     ++res.skipped;
 662                     out << endl;
 663                     // ignore the result of this test.
 664                     break;
 665             }
 666         }
 667     }
 668     return res;
 669 }
 670
 671 void
 672 test_driver::usage()
 673 {
 674     cout << "Usage: " << argv0 << " [-v|--verbose] [-o|--abort-on-error] " << opt_help
 675          << "[TESTNAME]..." << endl;
 676     cout << "       " << argv0 << " [-h|--help]" << endl;
 677     exit(1);
 678 }
 679
 680 /* Needs C linkage so we can pass it to atexit() without problems. */
 681 extern "C" {
 682 // Call upon program exit if there's more than one test run.
 683 static void
 684 report_totals(void)
 685 {
 686     test_driver::report(test_driver::total, "total");
 687 }
 688 }
 689
 690 void
 691 test_driver::report(const test_driver::result &r, const string &desc)
 692 {
 693     // Report totals at the end if we reported two or more subtotals.
 694     if (++runs == 2) atexit(report_totals);
 695
 696     if (r.succeeded != 0 || r.failed != 0) {
 697         cout << argv0 << " " << desc << ": ";
 698
 699         if (r.failed == 0)
 700             cout << "All ";
 701
 702         cout << col_green << r.succeeded << col_reset << " tests passed";
 703
 704         if (r.failed != 0)
 705             cout << ", " << col_red << r.failed << col_reset << " failed";
 706
 707         if (r.skipped) {
 708             cout << ", " << col_yellow << r.skipped << col_reset
 709                  << " skipped." << endl;
 710         } else {
 711             cout << "." << endl;
 712         }
 713     }
 714 }
 715
 716 void
 717 test_driver::add_command_line_option(const string &l, char s, string * arg)
 718 {
 719     short_opts.insert(make_pair(int(s), arg));
 720     opt_help += "[-";
 721     opt_help += s;
 722     opt_help += ' ';
 723     opt_help += l;
 724     opt_help += "] ";
 725 }
 726
 727 void
 728 test_driver::parse_command_line(int argc, char **argv)
 729 {
 730     argv0 = argv[0];
 731
 732 #ifdef HAVE_VALGRIND
 733     if (RUNNING_ON_VALGRIND) {
 734         if (getenv("XAPIAN_TESTSUITE_VALGRIND") != NULL) {
 735             // Open the valgrind log file, and unlink it.
 736             char fname[64];
 737             sprintf(fname, ".valgrind.log.%lu",
 738                     static_cast<unsigned long>(getpid()));
 739             vg_log_fd = open(fname, O_RDONLY|O_NONBLOCK|O_CLOEXEC);
 740             if (vg_log_fd != -1) unlink(fname);
 741         }
 742     }
 743 #endif
 744
 745 #ifndef __WIN32__
 746     {
 747         bool colourise = true;
 748         const char *p = getenv("XAPIAN_TESTSUITE_OUTPUT");
 749         if (p == NULL || !*p || strcmp(p, "auto") == 0) {
 750             colourise = isatty(1);
 751         } else if (strcmp(p, "plain") == 0) {
 752             colourise = false;
 753         }
 754         if (colourise) {
 755             col_red = "\x1b[1m\x1b[31m";
 756             col_green = "\x1b[1m\x1b[32m";
 757             col_yellow = "\x1b[1m\x1b[33m";
 758             col_reset = "\x1b[0m";
 759             use_cr = true;
 760         }
 761     }
 762 #endif
 763
 764     static const struct option long_opts[] = {
 765         {"verbose",             no_argument, 0, 'v'},
 766         {"abort-on-error",      no_argument, 0, 'o'},
 767         {"help",                no_argument, 0, 'h'},
 768         {NULL,                  0, 0, 0}
 769     };
 770
 771     string short_opts_string = "voh";
 772     map<int, string *>::const_iterator i;
 773     for (i = short_opts.begin(); i != short_opts.end(); ++i) {
 774         short_opts_string += char(i->first);
 775         short_opts_string += ':';
 776     }
 777     const char * opts = short_opts_string.c_str();
 778
 779     int c;
 780     while ((c = gnu_getopt_long(argc, argv, opts, long_opts, 0)) != -1) {
 781         switch (c) {
 782             case 'v':
 783                 ++verbose;
 784                 break;
 785             case 'o':
 786                 abort_on_error = true;
 787                 break;
 788             default: {
 789                 i = short_opts.find(c);
 790                 if (i != short_opts.end()) {
 791                     i->second->assign(optarg);
 792                     break;
 793                 }
 794                 // -h or unrecognised option
 795                 usage();
 796                 return; // usage() doesn't return ...
 797             }
 798         }
 799     }
 800
 801     if (verbose == 0) {
 802         const char *p = getenv("VERBOSE");
 803         if (p != NULL) {
 804             verbose = atoi(p);
 805         }
 806     }
 807
 808     while (argv[optind]) {
 809         test_names.push_back(string(argv[optind]));
 810         optind++;
 811     }
 812 }
 813
 814 int
 815 test_driver::run(const test_desc *tests)
 816 {
 817     test_driver driver(tests);
 818
 819     test_driver::result myresult;
 820     myresult = driver.run_tests(test_names.begin(), test_names.end());
 821
 822     subtotal += myresult;
 823
 824     return bool(myresult.failed); // if 0, then everything passed
 825 }
 826
 827 bool
 828 TEST_EQUAL_DOUBLE_(double a, double b)
 829 {
 830     if (a == b) return true;
 831     return (ceil(log10(max(fabs(a), fabs(b)))) - log10(fabs(a - b)) > DBL_DIG);
 832 }