xapian-core/tests/harness/testsuite.cc

   1 /* testsuite.cc: a test suite engine
   2  *
   3  * Copyright 1999,2000,2001 BrightStation PLC
   4  * Copyright 2002 Ananova Ltd
   5  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2012,2013,2015,2016,2017 Olly Betts
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  20  * USA
  21  */
  22
  23 #include <config.h>
  24
  25 #include "testsuite.h"
  26
  27 #ifndef NO_LIBXAPIAN
  28 # include "backendmanager.h"
  29 #endif
  30 #include "fdtracker.h"
  31 #include "testrunner.h"
  32 #include "safeerrno.h"
  33 #include "safeunistd.h"
  34
  35 #ifdef HAVE_VALGRIND
  36 # include <valgrind/memcheck.h>
  37 # include <sys/types.h>
  38 # include "safefcntl.h"
  39 #endif
  40
  41 #include <algorithm>
  42 #include <iostream>
  43 #include <set>
  44
  45 #include <cfloat> // For DBL_DIG.
  46 #include <cmath> // For ceil, fabs, log10.
  47 #include <cstdio>
  48 #include <cstdlib>
  49 #include <cstring>
  50
  51 #include "gnu_getopt.h"
  52
  53 #include <setjmp.h>
  54 #include <signal.h>
  55
  56 #include <exception>
  57 #ifdef USE_RTTI
  58 # include <typeinfo>
  59 # ifdef HAVE_CXXABI_H
  60 #  include <cxxabi.h>
  61 # endif
  62 #endif
  63
  64 #ifndef NO_LIBXAPIAN
  65 # include <xapian/error.h>
  66 #endif
  67 #include "filetests.h"
  68 #include "stringutils.h"
  69
  70 using namespace std;
  71
  72 /// The global verbose flag.
  73 int verbose;
  74
  75 #ifdef HAVE_VALGRIND
  76 static int vg_log_fd = -1;
  77 #endif
  78
  79 #ifdef HAVE_SIGSETJMP
  80 # define SIGSETJMP(ENV, SAVESIGS) sigsetjmp(ENV, SAVESIGS)
  81 # define SIGLONGJMP(ENV, VAL) siglongjmp(ENV, VAL)
  82 # define SIGJMP_BUF sigjmp_buf
  83 #else
  84 # define SIGSETJMP(ENV, SAVESIGS) setjmp(ENV)
  85 # define SIGLONGJMP(ENV, VAL) longjmp(ENV, VAL)
  86 # define SIGJMP_BUF jmp_buf
  87 #endif
  88
  89 /// The exception type we were expecting in TEST_EXCEPTION
  90 //  We use this to attempt to diagnose when the code fails to catch an
  91 //  exception when it should (due to a compiler or runtime fault in
  92 //  GCC 2.95 it seems)
  93 const char * expected_exception = NULL;
  94
  95 /// The debug printing stream
  96 std::ostringstream tout;
  97
  98 int test_driver::runs = 0;
  99 test_driver::result test_driver::subtotal;
 100 test_driver::result test_driver::total;
 101 string test_driver::argv0;
 102 string test_driver::opt_help;
 103 map<int, string *> test_driver::short_opts;
 104 vector<string> test_driver::test_names;
 105 bool test_driver::abort_on_error = false;
 106 string test_driver::col_red, test_driver::col_green;
 107 string test_driver::col_yellow, test_driver::col_reset;
 108 bool test_driver::use_cr = false;
 109
 110 void
 111 test_driver::write_and_clear_tout()
 112 {
 113     const string & s = tout.str();
 114     if (!s.empty()) {
 115         out << '\n' << s;
 116         tout.str(string());
 117     }
 118 }
 119
 120 string
 121 test_driver::get_srcdir()
 122 {
 123     char *p = getenv("srcdir");
 124     if (p != NULL) return string(p);
 125
 126 #ifdef __WIN32__
 127     // The path on argv[0] will always use \ for the directory separator.
 128     const char ARGV0_SEP = '\\';
 129 #else
 130     const char ARGV0_SEP = '/';
 131 #endif
 132     // Default srcdir to the pathname of argv[0].
 133     string srcdir(argv0);
 134     string::size_type i = srcdir.find_last_of(ARGV0_SEP);
 135     string srcfile;
 136     if (i != string::npos) {
 137         srcfile.assign(srcdir, i + 1, string::npos);
 138         srcdir.erase(i);
 139         // libtool may put the real executable in .libs.
 140         i = srcdir.find_last_of(ARGV0_SEP);
 141         if (srcdir.substr(i + 1) == ".libs") {
 142             srcdir.erase(i);
 143             // And it may have an "lt-" prefix.
 144             if (startswith(srcfile, "lt-")) srcfile.erase(0, 3);
 145         }
 146     } else {
 147         // No path of argv[0], so default srcdir to the current directory.
 148         // This may not work if libtool is involved as the true executable is
 149         // sometimes in ".libs".
 150         srcfile = srcdir;
 151         srcdir = ".";
 152     }
 153
 154     // Remove any trailing ".exe" suffix, since some platforms add this.
 155     if (endswith(srcfile, ".exe")) srcfile.resize(srcfile.size() - 4);
 156
 157     // Sanity check.
 158     if (!file_exists(srcdir + '/' + srcfile + ".cc")) {
 159         cout << argv0
 160              << ": srcdir is not in the environment and I can't guess it!\n"
 161                 "Run test programs using the runtest script - see HACKING for details"
 162              << endl;
 163         exit(1);
 164     }
 165     return srcdir;
 166 }
 167
 168 test_driver::test_driver(const test_desc *tests_)
 169         : out(cout.rdbuf()), tests(tests_)
 170 {
 171 }
 172
 173 static SIGJMP_BUF jb;
 174 static int signum = 0;
 175 static void * sigaddr = NULL;
 176
 177 // Needs C linkage so we can pass it to sigaction()/signal() without problems.
 178 extern "C" {
 179
 180 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 181 [[noreturn]]
 182 static void handle_sig(int signum_, siginfo_t *si, void *)
 183 {
 184     // Disable all our signal handlers to avoid problems if the signal
 185     // handling code causes a signal.
 186     struct sigaction sa;
 187     sa.sa_handler = SIG_DFL;
 188     sigemptyset(&sa.sa_mask);
 189     sa.sa_flags = 0;
 190     // We set the handlers with SA_RESETHAND, but that will only reset the
 191     // handler for the signal which fired.
 192     if (signum_ != SIGSEGV) sigaction(SIGSEGV, &sa, NULL);
 193     if (signum_ != SIGFPE) sigaction(SIGFPE, &sa, NULL);
 194     if (signum_ != SIGILL) sigaction(SIGILL, &sa, NULL);
 195 # ifdef SIGBUS
 196     if (signum_ != SIGBUS) sigaction(SIGBUS, &sa, NULL);
 197 # endif
 198 # ifdef SIGSTKFLT
 199     if (signum_ != SIGSTKFLT) sigaction(SIGSTKFLT, &sa, NULL);
 200 # endif
 201     signum = signum_;
 202     sigaddr = si->si_addr;
 203     SIGLONGJMP(jb, 1);
 204 }
 205
 206 #else
 207
 208 [[noreturn]]
 209 static void handle_sig(int signum_)
 210 {
 211     // Disable all our signal handlers to avoid problems if the signal
 212     // handling code causes a signal.
 213     signal(SIGSEGV, SIG_DFL);
 214     signal(SIGFPE, SIG_DFL);
 215     signal(SIGILL, SIG_DFL);
 216 #ifdef SIGBUS
 217     signal(SIGBUS, SIG_DFL);
 218 #endif
 219 #ifdef SIGSTKFLT
 220     signal(SIGSTKFLT, SIG_DFL);
 221 #endif
 222     signum = signum_;
 223     SIGLONGJMP(jb, 1);
 224 }
 225 #endif
 226
 227 }
 228
 229 class SignalRedirector {
 230   private:
 231     bool active;
 232   public:
 233     SignalRedirector() : active(false) { }
 234     void activate() {
 235         active = true;
 236         signum = 0;
 237         sigaddr = NULL;
 238         // SA_SIGINFO not universal (e.g. not present on Linux < 2.2 and Hurd).
 239 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 240         struct sigaction sa;
 241         sa.sa_sigaction = handle_sig;
 242         sigemptyset(&sa.sa_mask);
 243         sa.sa_flags = SA_RESETHAND|SA_SIGINFO;
 244         sigaction(SIGSEGV, &sa, NULL);
 245         sigaction(SIGFPE, &sa, NULL);
 246         sigaction(SIGILL, &sa, NULL);
 247 # ifdef SIGBUS
 248         sigaction(SIGBUS, &sa, NULL);
 249 # endif
 250 # ifdef SIGSTKFLT
 251         sigaction(SIGSTKFLT, &sa, NULL);
 252 # endif
 253 #else
 254         signal(SIGSEGV, handle_sig);
 255         signal(SIGFPE, handle_sig);
 256         signal(SIGILL, handle_sig);
 257 # ifdef SIGBUS
 258         signal(SIGBUS, handle_sig);
 259 # endif
 260 # ifdef SIGSTKFLT
 261         signal(SIGSTKFLT, handle_sig);
 262 # endif
 263 #endif
 264     }
 265     ~SignalRedirector() {
 266         if (active) {
 267 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 268             struct sigaction sa;
 269             sa.sa_handler = SIG_DFL;
 270             sigemptyset(&sa.sa_mask);
 271             sa.sa_flags = 0;
 272             sigaction(SIGSEGV, &sa, NULL);
 273             sigaction(SIGFPE, &sa, NULL);
 274             sigaction(SIGILL, &sa, NULL);
 275 # ifdef SIGBUS
 276             sigaction(SIGBUS, &sa, NULL);
 277 # endif
 278 # ifdef SIGSTKFLT
 279             sigaction(SIGSTKFLT, &sa, NULL);
 280 # endif
 281 #else
 282             signal(SIGSEGV, SIG_DFL);
 283             signal(SIGFPE, SIG_DFL);
 284             signal(SIGILL, SIG_DFL);
 285 # ifdef SIGBUS
 286             signal(SIGBUS, SIG_DFL);
 287 # endif
 288 # ifdef SIGSTKFLT
 289             signal(SIGSTKFLT, SIG_DFL);
 290 # endif
 291 #endif
 292         }
 293     }
 294 };
 295
 296 //  A wrapper around the tests to trap exceptions,
 297 //  and avoid having to catch them in every test function.
 298 //  If this test driver is used for anything other than
 299 //  Xapian tests, then this ought to be provided by
 300 //  the client, really.
 301 //  return: test_driver::PASS, test_driver::FAIL, or test_driver::SKIP
 302 test_driver::test_result
 303 test_driver::runtest(const test_desc *test)
 304 {
 305     // This is used to make a note of how many times we've run the test
 306     volatile int runcount = 0;
 307
 308     FDTracker fdtracker;
 309     fdtracker.init();
 310
 311     while (true) {
 312         tout.str(string());
 313         if (SIGSETJMP(jb, 1) == 0) {
 314             SignalRedirector sig; // use object so signal handlers are reset
 315             static bool catch_signals =
 316                 (getenv("XAPIAN_TESTSUITE_SIG_DFL") == NULL);
 317             if (catch_signals) sig.activate();
 318             try {
 319                 expected_exception = NULL;
 320 #ifdef HAVE_VALGRIND
 321                 int vg_errs = 0;
 322                 long vg_leaks = 0, vg_dubious = 0, vg_reachable = 0;
 323                 if (vg_log_fd != -1) {
 324                     VALGRIND_DO_LEAK_CHECK;
 325                     vg_errs = VALGRIND_COUNT_ERRORS;
 326                     long dummy;
 327                     VALGRIND_COUNT_LEAKS(vg_leaks, vg_dubious, vg_reachable, dummy);
 328                     (void)dummy;
 329                     // Skip past any unread log output.
 330                     lseek(vg_log_fd, 0, SEEK_END);
 331                 }
 332 #endif
 333                 if (!test->run()) {
 334                     out << col_red << " FAILED" << col_reset;
 335                     write_and_clear_tout();
 336                     return FAIL;
 337                 }
 338                 if (verbose > 1)
 339                     write_and_clear_tout();
 340 #ifndef NO_LIBXAPIAN
 341                 if (backendmanager)
 342                     backendmanager->clean_up();
 343 #endif
 344 #ifdef HAVE_VALGRIND
 345                 if (vg_log_fd != -1) {
 346                     // We must empty tout before asking valgrind to perform its
 347                     // leak checks, otherwise the buffers holding the output
 348                     // may be identified as a memory leak (especially if >1K of
 349                     // output has been buffered it appears...)
 350                     tout.str(string());
 351 #define REPORT_FAIL_VG(M) do { \
 352     if (verbose) { \
 353         while (true) { \
 354             ssize_t c = read(vg_log_fd, buf, sizeof(buf)); \
 355             if (c == 0 || (c < 0 && errno != EINTR)) break; \
 356             if (c > 0) out << string(buf, c); \
 357         } \
 358     } \
 359     out << " " << col_red << M << col_reset; \
 360 } while (0)
 361                     // Record the current position so we can restore it so
 362                     // REPORT_FAIL_VG() gets the whole output.
 363                     off_t curpos = lseek(vg_log_fd, 0, SEEK_CUR);
 364                     char buf[4096];
 365                     while (true) {
 366                         ssize_t c = read(vg_log_fd, buf, sizeof(buf));
 367                         if (c == 0 || (c < 0 && errno != EINTR)) {
 368                             buf[0] = 0;
 369                             break;
 370                         }
 371                         if (c > 0) {
 372                             // Valgrind output has "==<pid>== \n" between
 373                             // report "records", so skip any lines like that,
 374                             // and also any warnings and continuation lines.
 375                             ssize_t i = 0;
 376                             while (true) {
 377                                 const char * spc;
 378                                 spc = static_cast<const char *>(
 379                                         memchr(buf + i, ' ', c - i));
 380                                 if (!spc) {
 381                                     i = c;
 382                                     break;
 383                                 }
 384                                 i = spc - buf;
 385                                 if (++i >= c) break;
 386                                 if (buf[i] == '\n')
 387                                     continue;
 388                                 if (c - i >= 8 &&
 389                                     (memcmp(buf + i, "Warning:", 8) == 0 ||
 390                                      memcmp(buf + i, "   ", 3) == 0)) {
 391                                     // Skip this line.
 392                                     i += 3;
 393                                     const char * nl;
 394                                     nl = static_cast<const char *>(
 395                                             memchr(buf + i, '\n', c - i));
 396                                     if (!nl) {
 397                                         i = c;
 398                                         break;
 399                                     }
 400                                     i = nl - buf;
 401                                     continue;
 402                                 }
 403                                 break;
 404                             }
 405
 406                             char *start = buf + i;
 407                             c -= i;
 408                             if (c > 128) c = 128;
 409
 410                             {
 411                                 const char *p;
 412                                 p = static_cast<const char*>(
 413                                         memchr(start, '\n', c));
 414                                 if (p != NULL) c = p - start;
 415                             }
 416
 417                             memmove(buf, start, c);
 418                             buf[c] = '\0';
 419                             break;
 420                         }
 421                     }
 422                     lseek(vg_log_fd, curpos, SEEK_SET);
 423
 424                     int vg_errs2 = VALGRIND_COUNT_ERRORS;
 425                     vg_errs = vg_errs2 - vg_errs;
 426                     VALGRIND_DO_LEAK_CHECK;
 427                     long vg_leaks2 = 0, vg_dubious2 = 0, vg_reachable2 = 0;
 428                     long dummy;
 429                     VALGRIND_COUNT_LEAKS(vg_leaks2, vg_dubious2, vg_reachable2,
 430                                          dummy);
 431                     (void)dummy;
 432                     vg_leaks = vg_leaks2 - vg_leaks;
 433                     vg_dubious = vg_dubious2 - vg_dubious;
 434                     vg_reachable = vg_reachable2 - vg_reachable;
 435                     if (vg_errs) {
 436                         string fail_msg(buf);
 437                         if (fail_msg.empty())
 438                             fail_msg = "VALGRIND DETECTED A PROBLEM";
 439                         REPORT_FAIL_VG(fail_msg);
 440                         return FAIL;
 441                     }
 442                     if (vg_leaks > 0) {
 443                         REPORT_FAIL_VG("LEAKED " << vg_leaks << " BYTES");
 444                         return FAIL;
 445                     }
 446                     if (vg_dubious > 0) {
 447                         // If code deliberately holds onto blocks by a pointer
 448                         // not to the start (e.g. languages/utilities.c does)
 449                         // then we need to rerun the test to see if the leak is
 450                         // real...
 451                         if (runcount == 0) {
 452                             out << col_yellow << " PROBABLY LEAKED MEMORY - RETRYING TEST" << col_reset;
 453                             ++runcount;
 454                             // Ensure that any cached memory from fd tracking
 455                             // is allocated before we rerun the test.
 456                             (void)fdtracker.check();
 457                             continue;
 458                         }
 459                         REPORT_FAIL_VG("PROBABLY LEAKED " << vg_dubious << " BYTES");
 460                         return FAIL;
 461                     }
 462                     if (vg_reachable > 0) {
 463                         // C++ STL implementations often "horde" released
 464                         // memory - for GCC 3.4 and newer the runtest script
 465                         // sets GLIBCXX_FORCE_NEW=1 which will disable this
 466                         // behaviour so we avoid this issue, but for older
 467                         // GCC and other compilers this may be an issue.
 468                         //
 469                         // See also:
 470                         // http://valgrind.org/docs/FAQ/#faq.reports
 471                         //
 472                         // For now, just use runcount to rerun the test and see
 473                         // if more is leaked - hopefully this shouldn't give
 474                         // false positives.
 475                         if (runcount == 0) {
 476                             out << col_yellow << " POSSIBLE UNRELEASED MEMORY - RETRYING TEST" << col_reset;
 477                             ++runcount;
 478                             // Ensure that any cached memory from fd tracking
 479                             // is allocated before we rerun the test.
 480                             (void)fdtracker.check();
 481                             continue;
 482                         }
 483                         REPORT_FAIL_VG("FAILED TO RELEASE " << vg_reachable << " BYTES");
 484                         return FAIL;
 485                     }
 486                 }
 487 #endif
 488                 if (!fdtracker.check()) {
 489                     if (runcount == 0) {
 490                         out << col_yellow << " POSSIBLE FDLEAK:" << fdtracker.get_message() << col_reset;
 491                         ++runcount;
 492                         continue;
 493                     }
 494                     out << col_red << " FDLEAK:" << fdtracker.get_message() << col_reset;
 495                     return FAIL;
 496                 }
 497             } catch (const TestFail &) {
 498                 out << col_red << " FAILED" << col_reset;
 499                 write_and_clear_tout();
 500                 return FAIL;
 501             } catch (const TestSkip &) {
 502                 out << col_yellow << " SKIPPED" << col_reset;
 503                 write_and_clear_tout();
 504                 return SKIP;
 505 #ifndef NO_LIBXAPIAN
 506             } catch (const Xapian::Error &err) {
 507                 string errclass = err.get_type();
 508                 if (expected_exception && expected_exception == errclass) {
 509                     out << col_yellow << " C++ FAILED TO CATCH " << errclass << col_reset;
 510                     return SKIP;
 511                 }
 512                 if (errclass == "NetworkError" &&
 513                     err.get_error_string() != NULL &&
 514                     strcmp(err.get_error_string(), strerror(ECHILD)) == 0) {
 515                     // ECHILD suggests we've run out of processes, and that's
 516                     // much more likely to be a system issue than a Xapian bug.
 517                     //
 518                     // We also see apparently spurious ECHILD on Debian
 519                     // buildds sometimes: https://bugs.debian.org/681941
 520                     out << col_yellow << " ECHILD in network code" << col_reset;
 521                     return SKIP;
 522                 }
 523                 out << " " << col_red << err.get_description() << col_reset;
 524                 write_and_clear_tout();
 525                 return FAIL;
 526 #endif
 527             } catch (const string & msg) {
 528                 out << col_red << " EXCEPTION std::string " << msg << col_reset;
 529                 write_and_clear_tout();
 530                 return FAIL;
 531             } catch (const std::exception & e) {
 532                 out << " " << col_red;
 533 #ifndef USE_RTTI
 534                 out << "std::exception";
 535 #else
 536                 const char * name = typeid(e).name();
 537 # ifdef HAVE_CXXABI_H__
 538                 // __cxa_demangle() apparently requires GCC >= 3.1.
 539                 // Demangle the name which GCC returns for type_info::name().
 540                 int status;
 541                 char * realname = abi::__cxa_demangle(name, NULL, 0, &status);
 542                 if (realname) {
 543                     out << realname;
 544                     free(realname);
 545                 } else {
 546                     out << name;
 547                 }
 548 # else
 549                 out << name;
 550 # endif
 551 #endif
 552                 out << ": " << e.what() << col_reset;
 553                 write_and_clear_tout();
 554                 return FAIL;
 555             } catch (const char * msg) {
 556                 out << col_red;
 557                 if (msg) {
 558                     out << " EXCEPTION char * " << msg;
 559                 } else {
 560                     out << " EXCEPTION (char*)NULL";
 561                 }
 562                 out << col_reset;
 563                 write_and_clear_tout();
 564                 return FAIL;
 565             } catch (...) {
 566                 out << col_red << " UNKNOWN EXCEPTION" << col_reset;
 567                 write_and_clear_tout();
 568                 return FAIL;
 569             }
 570             return PASS;
 571         }
 572
 573         // Caught a signal.
 574         const char *signame = "SIGNAL";
 575         bool show_addr = true;
 576         switch (signum) {
 577             case SIGSEGV: signame = "SIGSEGV"; break;
 578             case SIGFPE: signame = "SIGFPE"; break;
 579             case SIGILL: signame = "SIGILL"; break;
 580 #ifdef SIGBUS
 581             case SIGBUS: signame = "SIGBUS"; break;
 582 #endif
 583 #ifdef SIGSTKFLT
 584             case SIGSTKFLT:
 585                 signame = "SIGSTKFLT";
 586                 show_addr = false;
 587                 break;
 588 #endif
 589         }
 590         out << " " << col_red << signame;
 591         if (show_addr) {
 592             char buf[40];
 593             sprintf(buf, " at %p", sigaddr);
 594             out << buf;
 595         }
 596         out << col_reset;
 597         write_and_clear_tout();
 598         return FAIL;
 599     }
 600 }
 601
 602 test_driver::result
 603 test_driver::run_tests(vector<string>::const_iterator b,
 604                        vector<string>::const_iterator e)
 605 {
 606     return do_run_tests(b, e);
 607 }
 608
 609 test_driver::result
 610 test_driver::run_tests()
 611 {
 612     const vector<string> blank;
 613     return do_run_tests(blank.begin(), blank.end());
 614 }
 615
 616 test_driver::result
 617 test_driver::do_run_tests(vector<string>::const_iterator b,
 618                           vector<string>::const_iterator e)
 619 {
 620     set<string> m(b, e);
 621     bool check_name = !m.empty();
 622
 623     test_driver::result res;
 624
 625     for (const test_desc *test = tests; test->name; ++test) {
 626         bool do_this_test = !check_name;
 627         if (!do_this_test && m.find(test->name) != m.end())
 628             do_this_test = true;
 629         if (!do_this_test) {
 630             // if this test is "foo123" see if "foo" was listed
 631             // this way "./testprog foo" can run foo1, foo2, etc.
 632             string t = test->name;
 633             string::size_type i;
 634             i = t.find_last_not_of("0123456789") + 1;
 635             if (i != string::npos) {
 636                 t.resize(i);
 637                 if (m.find(t) != m.end()) do_this_test = true;
 638             }
 639         }
 640         if (do_this_test) {
 641             out << "Running test: " << test->name << "...";
 642             out.flush();
 643             test_driver::test_result test_res = runtest(test);
 644 #ifndef NO_LIBXAPIAN
 645             if (backendmanager)
 646                 backendmanager->clean_up();
 647 #endif
 648             switch (test_res) {
 649                 case PASS:
 650                     ++res.succeeded;
 651                     if (verbose || !use_cr) {
 652                         out << col_green << " ok" << col_reset << endl;
 653                     } else {
 654                         out << "\r                                                                               \r";
 655                     }
 656                     break;
 657                 case FAIL:
 658                     ++res.failed;
 659                     out << endl;
 660                     if (abort_on_error) {
 661                         throw "Test failed - aborting further tests";
 662                     }
 663                     break;
 664                 case SKIP:
 665                     ++res.skipped;
 666                     out << endl;
 667                     // ignore the result of this test.
 668                     break;
 669             }
 670         }
 671     }
 672     return res;
 673 }
 674
 675 void
 676 test_driver::usage()
 677 {
 678     cout << "Usage: " << argv0 << " [-v|--verbose] [-o|--abort-on-error] " << opt_help
 679          << "[TESTNAME]..." << endl;
 680     cout << "       " << argv0 << " [-h|--help]" << endl;
 681     exit(1);
 682 }
 683
 684 /* Needs C linkage so we can pass it to atexit() without problems. */
 685 extern "C" {
 686 // Call upon program exit if there's more than one test run.
 687 static void
 688 report_totals(void)
 689 {
 690     test_driver::report(test_driver::total, "total");
 691 }
 692 }
 693
 694 void
 695 test_driver::report(const test_driver::result &r, const string &desc)
 696 {
 697     // Report totals at the end if we reported two or more subtotals.
 698     if (++runs == 2) atexit(report_totals);
 699
 700     if (r.succeeded != 0 || r.failed != 0) {
 701         cout << argv0 << " " << desc << ": ";
 702
 703         if (r.failed == 0)
 704             cout << "All ";
 705
 706         cout << col_green << r.succeeded << col_reset << " tests passed";
 707
 708         if (r.failed != 0)
 709             cout << ", " << col_red << r.failed << col_reset << " failed";
 710
 711         if (r.skipped) {
 712             cout << ", " << col_yellow << r.skipped << col_reset
 713                  << " skipped." << endl;
 714         } else {
 715             cout << "." << endl;
 716         }
 717     }
 718 }
 719
 720 void
 721 test_driver::add_command_line_option(const string &l, char s, string * arg)
 722 {
 723     short_opts.insert(make_pair(int(s), arg));
 724     opt_help += "[-";
 725     opt_help += s;
 726     opt_help += ' ';
 727     opt_help += l;
 728     opt_help += "] ";
 729 }
 730
 731 void
 732 test_driver::parse_command_line(int argc, char **argv)
 733 {
 734     argv0 = argv[0];
 735
 736 #ifdef HAVE_VALGRIND
 737     if (RUNNING_ON_VALGRIND) {
 738         if (getenv("XAPIAN_TESTSUITE_VALGRIND") != NULL) {
 739             // Open the valgrind log file, and unlink it.
 740             char fname[64];
 741             sprintf(fname, ".valgrind.log.%lu",
 742                     static_cast<unsigned long>(getpid()));
 743             vg_log_fd = open(fname, O_RDONLY|O_NONBLOCK|O_CLOEXEC);
 744             if (vg_log_fd != -1) unlink(fname);
 745         }
 746     }
 747 #endif
 748
 749 #ifndef __WIN32__
 750     {
 751         bool colourise = true;
 752         const char *p = getenv("XAPIAN_TESTSUITE_OUTPUT");
 753         if (p == NULL || !*p || strcmp(p, "auto") == 0) {
 754             colourise = isatty(1);
 755         } else if (strcmp(p, "plain") == 0) {
 756             colourise = false;
 757         }
 758         if (colourise) {
 759             col_red = "\x1b[1m\x1b[31m";
 760             col_green = "\x1b[1m\x1b[32m";
 761             col_yellow = "\x1b[1m\x1b[33m";
 762             col_reset = "\x1b[0m";
 763             use_cr = true;
 764         }
 765     }
 766 #endif
 767
 768     static const struct option long_opts[] = {
 769         {"verbose",             no_argument, 0, 'v'},
 770         {"abort-on-error",      no_argument, 0, 'o'},
 771         {"help",                no_argument, 0, 'h'},
 772         {NULL,                  0, 0, 0}
 773     };
 774
 775     string short_opts_string = "voh";
 776     map<int, string *>::const_iterator i;
 777     for (i = short_opts.begin(); i != short_opts.end(); ++i) {
 778         short_opts_string += char(i->first);
 779         short_opts_string += ':';
 780     }
 781     const char * opts = short_opts_string.c_str();
 782
 783     int c;
 784     while ((c = gnu_getopt_long(argc, argv, opts, long_opts, 0)) != -1) {
 785         switch (c) {
 786             case 'v':
 787                 ++verbose;
 788                 break;
 789             case 'o':
 790                 abort_on_error = true;
 791                 break;
 792             default: {
 793                 i = short_opts.find(c);
 794                 if (i != short_opts.end()) {
 795                     i->second->assign(optarg);
 796                     break;
 797                 }
 798                 // -h or unrecognised option
 799                 usage();
 800                 return; // usage() doesn't return ...
 801             }
 802         }
 803     }
 804
 805     if (verbose == 0) {
 806         const char *p = getenv("VERBOSE");
 807         if (p != NULL) {
 808             verbose = atoi(p);
 809         }
 810     }
 811
 812     while (argv[optind]) {
 813         test_names.push_back(string(argv[optind]));
 814         optind++;
 815     }
 816 }
 817
 818 int
 819 test_driver::run(const test_desc *tests)
 820 {
 821     test_driver driver(tests);
 822
 823     test_driver::result myresult;
 824     myresult = driver.run_tests(test_names.begin(), test_names.end());
 825
 826     subtotal += myresult;
 827
 828     return bool(myresult.failed); // if 0, then everything passed
 829 }
 830
 831 bool
 832 TEST_EQUAL_DOUBLE_(double a, double b)
 833 {
 834     if (a == b) return true;
 835     return (ceil(log10(max(fabs(a), fabs(b)))) - log10(fabs(a - b)) > DBL_DIG);
 836 }