xapian-core/tests/harness/testsuite.cc

   1 /* testsuite.cc: a test suite engine
   2  *
   3  * Copyright 1999,2000,2001 BrightStation PLC
   4  * Copyright 2002 Ananova Ltd
   5  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2012,2013,2015,2016,2017 Olly Betts
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  20  * USA
  21  */
  22
  23 #include <config.h>
  24
  25 #include "testsuite.h"
  26
  27 #ifndef NO_LIBXAPIAN
  28 # include "backendmanager.h"
  29 #endif
  30 #include "fdtracker.h"
  31 #include "testrunner.h"
  32 #include "safeerrno.h"
  33 #include "safeunistd.h"
  34
  35 #ifdef HAVE_VALGRIND
  36 # include <valgrind/memcheck.h>
  37 # include <sys/types.h>
  38 # include "safefcntl.h"
  39 #endif
  40
  41 #include <algorithm>
  42 #include <iostream>
  43 #include <set>
  44
  45 #include <cfloat> // For DBL_DIG.
  46 #include <cmath> // For ceil, fabs, log10.
  47 #include <cstdio>
  48 #include <cstdlib>
  49 #include <cstring>
  50
  51 #include "gnu_getopt.h"
  52
  53 #include <setjmp.h>
  54 #include <signal.h>
  55
  56 #include <exception>
  57 #ifdef USE_RTTI
  58 # include <typeinfo>
  59 # ifdef HAVE_CXXABI_H
  60 #  include <cxxabi.h>
  61 # endif
  62 #endif
  63
  64 #ifndef NO_LIBXAPIAN
  65 # include <xapian/error.h>
  66 #endif
  67 #include "filetests.h"
  68 #include "stringutils.h"
  69
  70 using namespace std;
  71
  72 /// The global verbose flag.
  73 int verbose;
  74
  75 #ifdef HAVE_VALGRIND
  76 static int vg_log_fd = -1;
  77 #endif
  78
  79 #ifdef HAVE_SIGSETJMP
  80 # define SIGSETJMP(ENV, SAVESIGS) sigsetjmp(ENV, SAVESIGS)
  81 # define SIGLONGJMP(ENV, VAL) siglongjmp(ENV, VAL)
  82 # define SIGJMP_BUF sigjmp_buf
  83 #else
  84 # define SIGSETJMP(ENV, SAVESIGS) setjmp(ENV)
  85 # define SIGLONGJMP(ENV, VAL) longjmp(ENV, VAL)
  86 # define SIGJMP_BUF jmp_buf
  87 #endif
  88
  89 /// The exception type we were expecting in TEST_EXCEPTION
  90 //  We use this to attempt to diagnose when the code fails to catch an
  91 //  exception when it should (due to a compiler or runtime fault in
  92 //  GCC 2.95 it seems)
  93 const char * expected_exception = NULL;
  94
  95 /// The debug printing stream
  96 std::ostringstream tout;
  97
  98 int test_driver::runs = 0;
  99 test_driver::result test_driver::subtotal;
 100 test_driver::result test_driver::total;
 101 string test_driver::argv0;
 102 string test_driver::opt_help;
 103 map<int, string *> test_driver::short_opts;
 104 vector<string> test_driver::test_names;
 105 bool test_driver::abort_on_error = false;
 106 string test_driver::col_red, test_driver::col_green;
 107 string test_driver::col_yellow, test_driver::col_reset;
 108 bool test_driver::use_cr = false;
 109
 110 void
 111 test_driver::write_and_clear_tout()
 112 {
 113     const string & s = tout.str();
 114     if (!s.empty()) {
 115         out << '\n' << s;
 116         tout.str(string());
 117     }
 118 }
 119
 120 string
 121 test_driver::get_srcdir()
 122 {
 123     char *p = getenv("srcdir");
 124     if (p != NULL) return string(p);
 125
 126     // Default srcdir to the pathname of argv[0].
 127     string srcdir(argv0);
 128     string::size_type i = srcdir.find_last_of(DIR_SEPS);
 129     string srcfile;
 130     if (i != string::npos) {
 131         srcfile.assign(srcdir, i + 1, string::npos);
 132         srcdir.erase(i);
 133         // libtool may put the real executable in .libs.
 134         i = srcdir.find_last_of(DIR_SEPS);
 135         if (srcdir.substr(i + 1) == ".libs") {
 136             srcdir.erase(i);
 137             // And it may have an "lt-" prefix.
 138             if (startswith(srcfile, "lt-")) srcfile.erase(0, 3);
 139         }
 140     } else {
 141         // No path of argv[0], so default srcdir to the current directory.
 142         // This may not work if libtool is involved as the true executable is
 143         // sometimes in ".libs".
 144         srcfile = srcdir;
 145         srcdir = ".";
 146     }
 147
 148     // Remove any trailing ".exe" suffix, since some platforms add this.
 149     if (endswith(srcfile, ".exe")) srcfile.resize(srcfile.size() - 4);
 150
 151     // Sanity check.
 152     if (!file_exists(srcdir + '/' + srcfile + ".cc")) {
 153         cout << argv0
 154              << ": srcdir is not in the environment and I can't guess it!\n"
 155                 "Run test programs using the runtest script - see HACKING for details"
 156              << endl;
 157         exit(1);
 158     }
 159     return srcdir;
 160 }
 161
 162 test_driver::test_driver(const test_desc *tests_)
 163         : out(cout.rdbuf()), tests(tests_)
 164 {
 165 }
 166
 167 static SIGJMP_BUF jb;
 168 static int signum = 0;
 169 static void * sigaddr = NULL;
 170
 171 // Needs C linkage so we can pass it to sigaction()/signal() without problems.
 172 extern "C" {
 173
 174 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 175 [[noreturn]]
 176 static void handle_sig(int signum_, siginfo_t *si, void *)
 177 {
 178     // Disable all our signal handlers to avoid problems if the signal
 179     // handling code causes a signal.
 180     struct sigaction sa;
 181     sa.sa_handler = SIG_DFL;
 182     sigemptyset(&sa.sa_mask);
 183     sa.sa_flags = 0;
 184     // We set the handlers with SA_RESETHAND, but that will only reset the
 185     // handler for the signal which fired.
 186     if (signum_ != SIGSEGV) sigaction(SIGSEGV, &sa, NULL);
 187     if (signum_ != SIGFPE) sigaction(SIGFPE, &sa, NULL);
 188     if (signum_ != SIGILL) sigaction(SIGILL, &sa, NULL);
 189 # ifdef SIGBUS
 190     if (signum_ != SIGBUS) sigaction(SIGBUS, &sa, NULL);
 191 # endif
 192 # ifdef SIGSTKFLT
 193     if (signum_ != SIGSTKFLT) sigaction(SIGSTKFLT, &sa, NULL);
 194 # endif
 195     signum = signum_;
 196     sigaddr = si->si_addr;
 197     SIGLONGJMP(jb, 1);
 198 }
 199
 200 #else
 201
 202 [[noreturn]]
 203 static void handle_sig(int signum_)
 204 {
 205     // Disable all our signal handlers to avoid problems if the signal
 206     // handling code causes a signal.
 207     signal(SIGSEGV, SIG_DFL);
 208     signal(SIGFPE, SIG_DFL);
 209     signal(SIGILL, SIG_DFL);
 210 #ifdef SIGBUS
 211     signal(SIGBUS, SIG_DFL);
 212 #endif
 213 #ifdef SIGSTKFLT
 214     signal(SIGSTKFLT, SIG_DFL);
 215 #endif
 216     signum = signum_;
 217     SIGLONGJMP(jb, 1);
 218 }
 219 #endif
 220
 221 }
 222
 223 class SignalRedirector {
 224   private:
 225     bool active;
 226   public:
 227     SignalRedirector() : active(false) { }
 228     void activate() {
 229         active = true;
 230         signum = 0;
 231         sigaddr = NULL;
 232         // SA_SIGINFO not universal (e.g. not present on Linux < 2.2 and Hurd).
 233 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 234         struct sigaction sa;
 235         sa.sa_sigaction = handle_sig;
 236         sigemptyset(&sa.sa_mask);
 237         sa.sa_flags = SA_RESETHAND|SA_SIGINFO;
 238         sigaction(SIGSEGV, &sa, NULL);
 239         sigaction(SIGFPE, &sa, NULL);
 240         sigaction(SIGILL, &sa, NULL);
 241 # ifdef SIGBUS
 242         sigaction(SIGBUS, &sa, NULL);
 243 # endif
 244 # ifdef SIGSTKFLT
 245         sigaction(SIGSTKFLT, &sa, NULL);
 246 # endif
 247 #else
 248         signal(SIGSEGV, handle_sig);
 249         signal(SIGFPE, handle_sig);
 250         signal(SIGILL, handle_sig);
 251 # ifdef SIGBUS
 252         signal(SIGBUS, handle_sig);
 253 # endif
 254 # ifdef SIGSTKFLT
 255         signal(SIGSTKFLT, handle_sig);
 256 # endif
 257 #endif
 258     }
 259     ~SignalRedirector() {
 260         if (active) {
 261 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 262             struct sigaction sa;
 263             sa.sa_handler = SIG_DFL;
 264             sigemptyset(&sa.sa_mask);
 265             sa.sa_flags = 0;
 266             sigaction(SIGSEGV, &sa, NULL);
 267             sigaction(SIGFPE, &sa, NULL);
 268             sigaction(SIGILL, &sa, NULL);
 269 # ifdef SIGBUS
 270             sigaction(SIGBUS, &sa, NULL);
 271 # endif
 272 # ifdef SIGSTKFLT
 273             sigaction(SIGSTKFLT, &sa, NULL);
 274 # endif
 275 #else
 276             signal(SIGSEGV, SIG_DFL);
 277             signal(SIGFPE, SIG_DFL);
 278             signal(SIGILL, SIG_DFL);
 279 # ifdef SIGBUS
 280             signal(SIGBUS, SIG_DFL);
 281 # endif
 282 # ifdef SIGSTKFLT
 283             signal(SIGSTKFLT, SIG_DFL);
 284 # endif
 285 #endif
 286         }
 287     }
 288 };
 289
 290 //  A wrapper around the tests to trap exceptions,
 291 //  and avoid having to catch them in every test function.
 292 //  If this test driver is used for anything other than
 293 //  Xapian tests, then this ought to be provided by
 294 //  the client, really.
 295 //  return: test_driver::PASS, test_driver::FAIL, or test_driver::SKIP
 296 test_driver::test_result
 297 test_driver::runtest(const test_desc *test)
 298 {
 299     // This is used to make a note of how many times we've run the test
 300     volatile int runcount = 0;
 301
 302     FDTracker fdtracker;
 303     fdtracker.init();
 304
 305     while (true) {
 306         tout.str(string());
 307         if (SIGSETJMP(jb, 1) == 0) {
 308             SignalRedirector sig; // use object so signal handlers are reset
 309             static bool catch_signals =
 310                 (getenv("XAPIAN_TESTSUITE_SIG_DFL") == NULL);
 311             if (catch_signals) sig.activate();
 312             try {
 313                 expected_exception = NULL;
 314 #ifdef HAVE_VALGRIND
 315                 int vg_errs = 0;
 316                 long vg_leaks = 0, vg_dubious = 0, vg_reachable = 0;
 317                 if (vg_log_fd != -1) {
 318                     VALGRIND_DO_LEAK_CHECK;
 319                     vg_errs = VALGRIND_COUNT_ERRORS;
 320                     long dummy;
 321                     VALGRIND_COUNT_LEAKS(vg_leaks, vg_dubious, vg_reachable, dummy);
 322                     (void)dummy;
 323                     // Skip past any unread log output.
 324                     lseek(vg_log_fd, 0, SEEK_END);
 325                 }
 326 #endif
 327                 if (!test->run()) {
 328                     out << col_red << " FAILED" << col_reset;
 329                     write_and_clear_tout();
 330                     return FAIL;
 331                 }
 332                 if (verbose > 1)
 333                     write_and_clear_tout();
 334 #ifndef NO_LIBXAPIAN
 335                 if (backendmanager)
 336                     backendmanager->clean_up();
 337 #endif
 338 #ifdef HAVE_VALGRIND
 339                 if (vg_log_fd != -1) {
 340                     // We must empty tout before asking valgrind to perform its
 341                     // leak checks, otherwise the buffers holding the output
 342                     // may be identified as a memory leak (especially if >1K of
 343                     // output has been buffered it appears...)
 344                     tout.str(string());
 345 #define REPORT_FAIL_VG(M) do { \
 346     if (verbose) { \
 347         while (true) { \
 348             ssize_t c = read(vg_log_fd, buf, sizeof(buf)); \
 349             if (c == 0 || (c < 0 && errno != EINTR)) break; \
 350             if (c > 0) out << string(buf, c); \
 351         } \
 352     } \
 353     out << " " << col_red << M << col_reset; \
 354 } while (0)
 355                     // Record the current position so we can restore it so
 356                     // REPORT_FAIL_VG() gets the whole output.
 357                     off_t curpos = lseek(vg_log_fd, 0, SEEK_CUR);
 358                     char buf[4096];
 359                     while (true) {
 360                         ssize_t c = read(vg_log_fd, buf, sizeof(buf));
 361                         if (c == 0 || (c < 0 && errno != EINTR)) {
 362                             buf[0] = 0;
 363                             break;
 364                         }
 365                         if (c > 0) {
 366                             // Valgrind output has "==<pid>== \n" between
 367                             // report "records", so skip any lines like that,
 368                             // and also any warnings and continuation lines.
 369                             ssize_t i = 0;
 370                             while (true) {
 371                                 const char * spc;
 372                                 spc = static_cast<const char *>(
 373                                         memchr(buf + i, ' ', c - i));
 374                                 if (!spc) {
 375                                     i = c;
 376                                     break;
 377                                 }
 378                                 i = spc - buf;
 379                                 if (++i >= c) break;
 380                                 if (buf[i] == '\n')
 381                                     continue;
 382                                 if (c - i >= 8 &&
 383                                     (memcmp(buf + i, "Warning:", 8) == 0 ||
 384                                      memcmp(buf + i, "   ", 3) == 0)) {
 385                                     // Skip this line.
 386                                     i += 3;
 387                                     const char * nl;
 388                                     nl = static_cast<const char *>(
 389                                             memchr(buf + i, '\n', c - i));
 390                                     if (!nl) {
 391                                         i = c;
 392                                         break;
 393                                     }
 394                                     i = nl - buf;
 395                                     continue;
 396                                 }
 397                                 break;
 398                             }
 399
 400                             char *start = buf + i;
 401                             c -= i;
 402                             if (c > 128) c = 128;
 403
 404                             {
 405                                 const char *p;
 406                                 p = static_cast<const char*>(
 407                                         memchr(start, '\n', c));
 408                                 if (p != NULL) c = p - start;
 409                             }
 410
 411                             memmove(buf, start, c);
 412                             buf[c] = '\0';
 413                             break;
 414                         }
 415                     }
 416                     lseek(vg_log_fd, curpos, SEEK_SET);
 417
 418                     int vg_errs2 = VALGRIND_COUNT_ERRORS;
 419                     vg_errs = vg_errs2 - vg_errs;
 420                     VALGRIND_DO_LEAK_CHECK;
 421                     long vg_leaks2 = 0, vg_dubious2 = 0, vg_reachable2 = 0;
 422                     long dummy;
 423                     VALGRIND_COUNT_LEAKS(vg_leaks2, vg_dubious2, vg_reachable2,
 424                                          dummy);
 425                     (void)dummy;
 426                     vg_leaks = vg_leaks2 - vg_leaks;
 427                     vg_dubious = vg_dubious2 - vg_dubious;
 428                     vg_reachable = vg_reachable2 - vg_reachable;
 429                     if (vg_errs) {
 430                         string fail_msg(buf);
 431                         if (fail_msg.empty())
 432                             fail_msg = "VALGRIND DETECTED A PROBLEM";
 433                         REPORT_FAIL_VG(fail_msg);
 434                         return FAIL;
 435                     }
 436                     if (vg_leaks > 0) {
 437                         REPORT_FAIL_VG("LEAKED " << vg_leaks << " BYTES");
 438                         return FAIL;
 439                     }
 440                     if (vg_dubious > 0) {
 441                         // If code deliberately holds onto blocks by a pointer
 442                         // not to the start (e.g. languages/utilities.c does)
 443                         // then we need to rerun the test to see if the leak is
 444                         // real...
 445                         if (runcount == 0) {
 446                             out << col_yellow << " PROBABLY LEAKED MEMORY - RETRYING TEST" << col_reset;
 447                             ++runcount;
 448                             // Ensure that any cached memory from fd tracking
 449                             // is allocated before we rerun the test.
 450                             (void)fdtracker.check();
 451                             continue;
 452                         }
 453                         REPORT_FAIL_VG("PROBABLY LEAKED " << vg_dubious << " BYTES");
 454                         return FAIL;
 455                     }
 456                     if (vg_reachable > 0) {
 457                         // C++ STL implementations often "horde" released
 458                         // memory - for GCC 3.4 and newer the runtest script
 459                         // sets GLIBCXX_FORCE_NEW=1 which will disable this
 460                         // behaviour so we avoid this issue, but for older
 461                         // GCC and other compilers this may be an issue.
 462                         //
 463                         // See also:
 464                         // http://valgrind.org/docs/FAQ/#faq.reports
 465                         //
 466                         // For now, just use runcount to rerun the test and see
 467                         // if more is leaked - hopefully this shouldn't give
 468                         // false positives.
 469                         if (runcount == 0) {
 470                             out << col_yellow << " POSSIBLE UNRELEASED MEMORY - RETRYING TEST" << col_reset;
 471                             ++runcount;
 472                             // Ensure that any cached memory from fd tracking
 473                             // is allocated before we rerun the test.
 474                             (void)fdtracker.check();
 475                             continue;
 476                         }
 477                         REPORT_FAIL_VG("FAILED TO RELEASE " << vg_reachable << " BYTES");
 478                         return FAIL;
 479                     }
 480                 }
 481 #endif
 482                 if (!fdtracker.check()) {
 483                     if (runcount == 0) {
 484                         out << col_yellow << " POSSIBLE FDLEAK:" << fdtracker.get_message() << col_reset;
 485                         ++runcount;
 486                         continue;
 487                     }
 488                     out << col_red << " FDLEAK:" << fdtracker.get_message() << col_reset;
 489                     return FAIL;
 490                 }
 491             } catch (const TestFail &) {
 492                 out << col_red << " FAILED" << col_reset;
 493                 write_and_clear_tout();
 494                 return FAIL;
 495             } catch (const TestSkip &) {
 496                 out << col_yellow << " SKIPPED" << col_reset;
 497                 write_and_clear_tout();
 498                 return SKIP;
 499 #ifndef NO_LIBXAPIAN
 500             } catch (const Xapian::Error &err) {
 501                 string errclass = err.get_type();
 502                 if (expected_exception && expected_exception == errclass) {
 503                     out << col_yellow << " C++ FAILED TO CATCH " << errclass << col_reset;
 504                     return SKIP;
 505                 }
 506                 if (errclass == "NetworkError" &&
 507                     err.get_error_string() != NULL &&
 508                     strcmp(err.get_error_string(), strerror(ECHILD)) == 0) {
 509                     // ECHILD suggests we've run out of processes, and that's
 510                     // much more likely to be a system issue than a Xapian bug.
 511                     //
 512                     // We also see apparently spurious ECHILD on Debian
 513                     // buildds sometimes: https://bugs.debian.org/681941
 514                     out << col_yellow << " ECHILD in network code" << col_reset;
 515                     return SKIP;
 516                 }
 517                 out << " " << col_red << err.get_description() << col_reset;
 518                 write_and_clear_tout();
 519                 return FAIL;
 520 #endif
 521             } catch (const string & msg) {
 522                 out << col_red << " EXCEPTION std::string " << msg << col_reset;
 523                 write_and_clear_tout();
 524                 return FAIL;
 525             } catch (const std::exception & e) {
 526                 out << " " << col_red;
 527 #ifndef USE_RTTI
 528                 out << "std::exception";
 529 #else
 530                 const char * name = typeid(e).name();
 531 # ifdef HAVE_CXXABI_H__
 532                 // __cxa_demangle() apparently requires GCC >= 3.1.
 533                 // Demangle the name which GCC returns for type_info::name().
 534                 int status;
 535                 char * realname = abi::__cxa_demangle(name, NULL, 0, &status);
 536                 if (realname) {
 537                     out << realname;
 538                     free(realname);
 539                 } else {
 540                     out << name;
 541                 }
 542 # else
 543                 out << name;
 544 # endif
 545 #endif
 546                 out << ": " << e.what() << col_reset;
 547                 write_and_clear_tout();
 548                 return FAIL;
 549             } catch (const char * msg) {
 550                 out << col_red;
 551                 if (msg) {
 552                     out << " EXCEPTION char * " << msg;
 553                 } else {
 554                     out << " EXCEPTION (char*)NULL";
 555                 }
 556                 out << col_reset;
 557                 write_and_clear_tout();
 558                 return FAIL;
 559             } catch (...) {
 560                 out << col_red << " UNKNOWN EXCEPTION" << col_reset;
 561                 write_and_clear_tout();
 562                 return FAIL;
 563             }
 564             return PASS;
 565         }
 566
 567         // Caught a signal.
 568         const char *signame = "SIGNAL";
 569         bool show_addr = true;
 570         switch (signum) {
 571             case SIGSEGV: signame = "SIGSEGV"; break;
 572             case SIGFPE: signame = "SIGFPE"; break;
 573             case SIGILL: signame = "SIGILL"; break;
 574 #ifdef SIGBUS
 575             case SIGBUS: signame = "SIGBUS"; break;
 576 #endif
 577 #ifdef SIGSTKFLT
 578             case SIGSTKFLT:
 579                 signame = "SIGSTKFLT";
 580                 show_addr = false;
 581                 break;
 582 #endif
 583         }
 584         out << " " << col_red << signame;
 585         if (show_addr) {
 586             char buf[40];
 587             sprintf(buf, " at %p", sigaddr);
 588             out << buf;
 589         }
 590         out << col_reset;
 591         write_and_clear_tout();
 592         return FAIL;
 593     }
 594 }
 595
 596 test_driver::result
 597 test_driver::run_tests(vector<string>::const_iterator b,
 598                        vector<string>::const_iterator e)
 599 {
 600     return do_run_tests(b, e);
 601 }
 602
 603 test_driver::result
 604 test_driver::run_tests()
 605 {
 606     const vector<string> blank;
 607     return do_run_tests(blank.begin(), blank.end());
 608 }
 609
 610 test_driver::result
 611 test_driver::do_run_tests(vector<string>::const_iterator b,
 612                           vector<string>::const_iterator e)
 613 {
 614     set<string> m(b, e);
 615     bool check_name = !m.empty();
 616
 617     test_driver::result res;
 618
 619     for (const test_desc *test = tests; test->name; ++test) {
 620         bool do_this_test = !check_name;
 621         if (!do_this_test && m.find(test->name) != m.end())
 622             do_this_test = true;
 623         if (!do_this_test) {
 624             // if this test is "foo123" see if "foo" was listed
 625             // this way "./testprog foo" can run foo1, foo2, etc.
 626             string t = test->name;
 627             string::size_type i;
 628             i = t.find_last_not_of("0123456789") + 1;
 629             if (i != string::npos) {
 630                 t.resize(i);
 631                 if (m.find(t) != m.end()) do_this_test = true;
 632             }
 633         }
 634         if (do_this_test) {
 635             out << "Running test: " << test->name << "...";
 636             out.flush();
 637             test_driver::test_result test_res = runtest(test);
 638 #ifndef NO_LIBXAPIAN
 639             if (backendmanager)
 640                 backendmanager->clean_up();
 641 #endif
 642             switch (test_res) {
 643                 case PASS:
 644                     ++res.succeeded;
 645                     if (verbose || !use_cr) {
 646                         out << col_green << " ok" << col_reset << endl;
 647                     } else {
 648                         out << "\r                                                                               \r";
 649                     }
 650                     break;
 651                 case FAIL:
 652                     ++res.failed;
 653                     out << endl;
 654                     if (abort_on_error) {
 655                         throw "Test failed - aborting further tests";
 656                     }
 657                     break;
 658                 case SKIP:
 659                     ++res.skipped;
 660                     out << endl;
 661                     // ignore the result of this test.
 662                     break;
 663             }
 664         }
 665     }
 666     return res;
 667 }
 668
 669 void
 670 test_driver::usage()
 671 {
 672     cout << "Usage: " << argv0 << " [-v|--verbose] [-o|--abort-on-error] " << opt_help
 673          << "[TESTNAME]..." << endl;
 674     cout << "       " << argv0 << " [-h|--help]" << endl;
 675     exit(1);
 676 }
 677
 678 /* Needs C linkage so we can pass it to atexit() without problems. */
 679 extern "C" {
 680 // Call upon program exit if there's more than one test run.
 681 static void
 682 report_totals(void)
 683 {
 684     test_driver::report(test_driver::total, "total");
 685 }
 686 }
 687
 688 void
 689 test_driver::report(const test_driver::result &r, const string &desc)
 690 {
 691     // Report totals at the end if we reported two or more subtotals.
 692     if (++runs == 2) atexit(report_totals);
 693
 694     if (r.succeeded != 0 || r.failed != 0) {
 695         cout << argv0 << " " << desc << ": ";
 696
 697         if (r.failed == 0)
 698             cout << "All ";
 699
 700         cout << col_green << r.succeeded << col_reset << " tests passed";
 701
 702         if (r.failed != 0)
 703             cout << ", " << col_red << r.failed << col_reset << " failed";
 704
 705         if (r.skipped) {
 706             cout << ", " << col_yellow << r.skipped << col_reset
 707                  << " skipped." << endl;
 708         } else {
 709             cout << "." << endl;
 710         }
 711     }
 712 }
 713
 714 void
 715 test_driver::add_command_line_option(const string &l, char s, string * arg)
 716 {
 717     short_opts.insert(make_pair(int(s), arg));
 718     opt_help += "[-";
 719     opt_help += s;
 720     opt_help += ' ';
 721     opt_help += l;
 722     opt_help += "] ";
 723 }
 724
 725 void
 726 test_driver::parse_command_line(int argc, char **argv)
 727 {
 728     argv0 = argv[0];
 729
 730 #ifdef HAVE_VALGRIND
 731     if (RUNNING_ON_VALGRIND) {
 732         if (getenv("XAPIAN_TESTSUITE_VALGRIND") != NULL) {
 733             // Open the valgrind log file, and unlink it.
 734             char fname[64];
 735             sprintf(fname, ".valgrind.log.%lu",
 736                     static_cast<unsigned long>(getpid()));
 737             vg_log_fd = open(fname, O_RDONLY|O_NONBLOCK|O_CLOEXEC);
 738             if (vg_log_fd != -1) unlink(fname);
 739         }
 740     }
 741 #endif
 742
 743 #ifndef __WIN32__
 744     {
 745         bool colourise = true;
 746         const char *p = getenv("XAPIAN_TESTSUITE_OUTPUT");
 747         if (p == NULL || !*p || strcmp(p, "auto") == 0) {
 748             colourise = isatty(1);
 749         } else if (strcmp(p, "plain") == 0) {
 750             colourise = false;
 751         }
 752         if (colourise) {
 753             col_red = "\x1b[1m\x1b[31m";
 754             col_green = "\x1b[1m\x1b[32m";
 755             col_yellow = "\x1b[1m\x1b[33m";
 756             col_reset = "\x1b[0m";
 757             use_cr = true;
 758         }
 759     }
 760 #endif
 761
 762     static const struct option long_opts[] = {
 763         {"verbose",             no_argument, 0, 'v'},
 764         {"abort-on-error",      no_argument, 0, 'o'},
 765         {"help",                no_argument, 0, 'h'},
 766         {NULL,                  0, 0, 0}
 767     };
 768
 769     string short_opts_string = "voh";
 770     map<int, string *>::const_iterator i;
 771     for (i = short_opts.begin(); i != short_opts.end(); ++i) {
 772         short_opts_string += char(i->first);
 773         short_opts_string += ':';
 774     }
 775     const char * opts = short_opts_string.c_str();
 776
 777     int c;
 778     while ((c = gnu_getopt_long(argc, argv, opts, long_opts, 0)) != -1) {
 779         switch (c) {
 780             case 'v':
 781                 ++verbose;
 782                 break;
 783             case 'o':
 784                 abort_on_error = true;
 785                 break;
 786             default: {
 787                 i = short_opts.find(c);
 788                 if (i != short_opts.end()) {
 789                     i->second->assign(optarg);
 790                     break;
 791                 }
 792                 // -h or unrecognised option
 793                 usage();
 794                 return; // usage() doesn't return ...
 795             }
 796         }
 797     }
 798
 799     if (verbose == 0) {
 800         const char *p = getenv("VERBOSE");
 801         if (p != NULL) {
 802             verbose = atoi(p);
 803         }
 804     }
 805
 806     while (argv[optind]) {
 807         test_names.push_back(string(argv[optind]));
 808         optind++;
 809     }
 810 }
 811
 812 int
 813 test_driver::run(const test_desc *tests)
 814 {
 815     test_driver driver(tests);
 816
 817     test_driver::result myresult;
 818     myresult = driver.run_tests(test_names.begin(), test_names.end());
 819
 820     subtotal += myresult;
 821
 822     return bool(myresult.failed); // if 0, then everything passed
 823 }
 824
 825 bool
 826 TEST_EQUAL_DOUBLE_(double a, double b)
 827 {
 828     if (a == b) return true;
 829     return (ceil(log10(max(fabs(a), fabs(b)))) - log10(fabs(a - b)) > DBL_DIG);
 830 }