xapian-core/tests/harness/testsuite.cc

   1 /* testsuite.cc: a test suite engine
   2  *
   3  * Copyright 1999,2000,2001 BrightStation PLC
   4  * Copyright 2002 Ananova Ltd
   5  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2012,2013,2015,2016,2017 Olly Betts
   6  * Copyright 2007 Richard Boulton
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License as
  10  * published by the Free Software Foundation; either version 2 of the
  11  * License, or (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  21  * USA
  22  */
  23
  24 #include <config.h>
  25
  26 #include "testsuite.h"
  27
  28 #ifndef NO_LIBXAPIAN
  29 # include "backendmanager.h"
  30 #endif
  31 #include "fdtracker.h"
  32 #include "testrunner.h"
  33 #include "safeunistd.h"
  34
  35 #ifdef HAVE_VALGRIND
  36 # include <valgrind/memcheck.h>
  37 # include <sys/types.h>
  38 # include "safefcntl.h"
  39 #endif
  40
  41 #include <algorithm>
  42 #include <iomanip>
  43 #include <iostream>
  44 #include <set>
  45
  46 #include <cerrno>
  47 #include <cfloat> // For DBL_DIG.
  48 #include <cmath> // For ceil, fabs, log10.
  49 #include <cstdio>
  50 #include <cstdlib>
  51 #include <cstring>
  52
  53 #include "gnu_getopt.h"
  54
  55 #include <setjmp.h>
  56 #include <signal.h>
  57
  58 #include <exception>
  59 #ifdef USE_RTTI
  60 # include <typeinfo>
  61 # ifdef HAVE_CXXABI_H
  62 #  include <cxxabi.h>
  63 # endif
  64 #endif
  65
  66 #ifndef NO_LIBXAPIAN
  67 # include <xapian/error.h>
  68 #endif
  69 #include "filetests.h"
  70 #include "stringutils.h"
  71
  72 using namespace std;
  73
  74 /// The global verbose flag.
  75 int verbose;
  76
  77 #ifdef HAVE_VALGRIND
  78 static int vg_log_fd = -1;
  79 #endif
  80
  81 #ifdef HAVE_SIGSETJMP
  82 # define SIGSETJMP(ENV, SAVESIGS) sigsetjmp(ENV, SAVESIGS)
  83 # define SIGLONGJMP(ENV, VAL) siglongjmp(ENV, VAL)
  84 # define SIGJMP_BUF sigjmp_buf
  85 #else
  86 # define SIGSETJMP(ENV, SAVESIGS) setjmp(ENV)
  87 # define SIGLONGJMP(ENV, VAL) longjmp(ENV, VAL)
  88 # define SIGJMP_BUF jmp_buf
  89 #endif
  90
  91 /// The exception type we were expecting in TEST_EXCEPTION
  92 //  We use this to attempt to diagnose when the code fails to catch an
  93 //  exception when it should (due to a compiler or runtime fault in
  94 //  GCC 2.95 it seems)
  95 const char * expected_exception = NULL;
  96
  97 const char* expected_failure;
  98
  99 /// The debug printing stream
 100 std::ostringstream tout;
 101
 102 int test_driver::runs = 0;
 103 test_driver::result test_driver::subtotal;
 104 test_driver::result test_driver::total;
 105 string test_driver::argv0;
 106 string test_driver::opt_help;
 107 map<int, string *> test_driver::short_opts;
 108 vector<string> test_driver::test_names;
 109 bool test_driver::abort_on_error = false;
 110 string test_driver::col_red, test_driver::col_green;
 111 string test_driver::col_yellow, test_driver::col_reset;
 112 bool test_driver::use_cr = false;
 113
 114 void
 115 test_driver::write_and_clear_tout()
 116 {
 117     const string & s = tout.str();
 118     if (!s.empty()) {
 119         out << '\n' << s;
 120         tout.str(string());
 121     }
 122 }
 123
 124 string
 125 test_driver::get_srcdir()
 126 {
 127     char *p = getenv("srcdir");
 128     if (p != NULL) return string(p);
 129
 130     // Default srcdir to the pathname of argv[0].
 131     string srcdir(argv0);
 132     string::size_type i = srcdir.find_last_of(DIR_SEPS);
 133     string srcfile;
 134     if (i != string::npos) {
 135         srcfile.assign(srcdir, i + 1, string::npos);
 136         srcdir.erase(i);
 137         // libtool may put the real executable in .libs.
 138         i = srcdir.find_last_of(DIR_SEPS);
 139         if (srcdir.substr(i + 1) == ".libs") {
 140             srcdir.erase(i);
 141             // And it may have an "lt-" prefix.
 142             if (startswith(srcfile, "lt-")) srcfile.erase(0, 3);
 143         }
 144     } else {
 145         // No path of argv[0], so default srcdir to the current directory.
 146         // This may not work if libtool is involved as the true executable is
 147         // sometimes in ".libs".
 148         srcfile = srcdir;
 149         srcdir = ".";
 150     }
 151
 152     // Remove any trailing ".exe" suffix, since some platforms add this.
 153     if (endswith(srcfile, ".exe")) srcfile.resize(srcfile.size() - 4);
 154
 155     // Sanity check.
 156     if (!file_exists(srcdir + '/' + srcfile + ".cc")) {
 157         cout << argv0
 158              << ": srcdir is not in the environment and I can't guess it!\n"
 159                 "Run test programs using the runtest script - see HACKING for details"
 160              << endl;
 161         exit(1);
 162     }
 163     return srcdir;
 164 }
 165
 166 test_driver::test_driver(const test_desc *tests_)
 167         : out(cout.rdbuf()), tests(tests_)
 168 {
 169     tout << boolalpha;
 170 }
 171
 172 static SIGJMP_BUF jb;
 173 static int signum = 0;
 174 static void * sigaddr = NULL;
 175
 176 // Needs C linkage so we can pass it to sigaction()/signal() without problems.
 177 extern "C" {
 178
 179 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 180 [[noreturn]]
 181 static void handle_sig(int signum_, siginfo_t *si, void *)
 182 {
 183     // Disable all our signal handlers to avoid problems if the signal
 184     // handling code causes a signal.
 185     struct sigaction sa;
 186     sa.sa_handler = SIG_DFL;
 187     sigemptyset(&sa.sa_mask);
 188     sa.sa_flags = 0;
 189     // We set the handlers with SA_RESETHAND, but that will only reset the
 190     // handler for the signal which fired.
 191     if (signum_ != SIGSEGV) sigaction(SIGSEGV, &sa, NULL);
 192     if (signum_ != SIGFPE) sigaction(SIGFPE, &sa, NULL);
 193     if (signum_ != SIGILL) sigaction(SIGILL, &sa, NULL);
 194 # ifdef SIGBUS
 195     if (signum_ != SIGBUS) sigaction(SIGBUS, &sa, NULL);
 196 # endif
 197 # ifdef SIGSTKFLT
 198     if (signum_ != SIGSTKFLT) sigaction(SIGSTKFLT, &sa, NULL);
 199 # endif
 200     signum = signum_;
 201     sigaddr = si->si_addr;
 202     SIGLONGJMP(jb, 1);
 203 }
 204
 205 #else
 206
 207 [[noreturn]]
 208 static void handle_sig(int signum_)
 209 {
 210     // Disable all our signal handlers to avoid problems if the signal
 211     // handling code causes a signal.
 212     signal(SIGSEGV, SIG_DFL);
 213     signal(SIGFPE, SIG_DFL);
 214     signal(SIGILL, SIG_DFL);
 215 #ifdef SIGBUS
 216     signal(SIGBUS, SIG_DFL);
 217 #endif
 218 #ifdef SIGSTKFLT
 219     signal(SIGSTKFLT, SIG_DFL);
 220 #endif
 221     signum = signum_;
 222     SIGLONGJMP(jb, 1);
 223 }
 224 #endif
 225
 226 }
 227
 228 class SignalRedirector {
 229   private:
 230     bool active;
 231   public:
 232     SignalRedirector() : active(false) { }
 233     void activate() {
 234         active = true;
 235         signum = 0;
 236         sigaddr = NULL;
 237         // SA_SIGINFO not universal (e.g. not present on Linux < 2.2 and Hurd).
 238 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 239         struct sigaction sa;
 240         sa.sa_sigaction = handle_sig;
 241         sigemptyset(&sa.sa_mask);
 242         sa.sa_flags = SA_RESETHAND|SA_SIGINFO;
 243         sigaction(SIGSEGV, &sa, NULL);
 244         sigaction(SIGFPE, &sa, NULL);
 245         sigaction(SIGILL, &sa, NULL);
 246 # ifdef SIGBUS
 247         sigaction(SIGBUS, &sa, NULL);
 248 # endif
 249 # ifdef SIGSTKFLT
 250         sigaction(SIGSTKFLT, &sa, NULL);
 251 # endif
 252 #else
 253         signal(SIGSEGV, handle_sig);
 254         signal(SIGFPE, handle_sig);
 255         signal(SIGILL, handle_sig);
 256 # ifdef SIGBUS
 257         signal(SIGBUS, handle_sig);
 258 # endif
 259 # ifdef SIGSTKFLT
 260         signal(SIGSTKFLT, handle_sig);
 261 # endif
 262 #endif
 263     }
 264     ~SignalRedirector() {
 265         if (active) {
 266 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 267             struct sigaction sa;
 268             sa.sa_handler = SIG_DFL;
 269             sigemptyset(&sa.sa_mask);
 270             sa.sa_flags = 0;
 271             sigaction(SIGSEGV, &sa, NULL);
 272             sigaction(SIGFPE, &sa, NULL);
 273             sigaction(SIGILL, &sa, NULL);
 274 # ifdef SIGBUS
 275             sigaction(SIGBUS, &sa, NULL);
 276 # endif
 277 # ifdef SIGSTKFLT
 278             sigaction(SIGSTKFLT, &sa, NULL);
 279 # endif
 280 #else
 281             signal(SIGSEGV, SIG_DFL);
 282             signal(SIGFPE, SIG_DFL);
 283             signal(SIGILL, SIG_DFL);
 284 # ifdef SIGBUS
 285             signal(SIGBUS, SIG_DFL);
 286 # endif
 287 # ifdef SIGSTKFLT
 288             signal(SIGSTKFLT, SIG_DFL);
 289 # endif
 290 #endif
 291         }
 292     }
 293 };
 294
 295 //  A wrapper around the tests to trap exceptions,
 296 //  and avoid having to catch them in every test function.
 297 //  If this test driver is used for anything other than
 298 //  Xapian tests, then this ought to be provided by
 299 //  the client, really.
 300 //  return: test_driver::PASS, test_driver::FAIL, test_driver::SKIP,
 301 //  test_driver::XFAIL or test_driver:XPASS.
 302 test_driver::test_result
 303 test_driver::runtest(const test_desc *test)
 304 {
 305     // This is used to make a note of how many times we've run the test
 306     volatile int runcount = 0;
 307
 308     FDTracker fdtracker;
 309     fdtracker.init();
 310
 311     while (true) {
 312         tout.str(string());
 313         if (SIGSETJMP(jb, 1) == 0) {
 314             SignalRedirector sig; // use object so signal handlers are reset
 315             static bool catch_signals =
 316                 (getenv("XAPIAN_TESTSUITE_SIG_DFL") == NULL);
 317             if (catch_signals) sig.activate();
 318             try {
 319                 expected_exception = NULL;
 320                 expected_failure = NULL;
 321 #ifdef HAVE_VALGRIND
 322                 int vg_errs = 0;
 323                 long vg_leaks = 0, vg_dubious = 0, vg_reachable = 0;
 324                 if (vg_log_fd != -1) {
 325                     VALGRIND_DO_LEAK_CHECK;
 326                     vg_errs = VALGRIND_COUNT_ERRORS;
 327                     long dummy;
 328                     VALGRIND_COUNT_LEAKS(vg_leaks, vg_dubious, vg_reachable, dummy);
 329                     (void)dummy;
 330                     // Skip past any unread log output.
 331                     lseek(vg_log_fd, 0, SEEK_END);
 332                 }
 333 #endif
 334                 if (!test->run()) {
 335                     out << ' ';
 336                     if (expected_failure) {
 337                         out << col_yellow << "XFAIL (" << expected_failure << ")";
 338                     } else {
 339                         out << col_red << "FAILED";
 340                     }
 341                     out << col_reset;
 342                     write_and_clear_tout();
 343                     return expected_failure ? XFAIL : FAIL;
 344                 }
 345                 if (verbose > 1)
 346                     write_and_clear_tout();
 347 #ifndef NO_LIBXAPIAN
 348                 if (backendmanager)
 349                     backendmanager->clean_up();
 350 #endif
 351 #ifdef HAVE_VALGRIND
 352                 if (vg_log_fd != -1) {
 353                     // We must empty tout before asking valgrind to perform its
 354                     // leak checks, otherwise the buffers holding the output
 355                     // may be identified as a memory leak (especially if >1K of
 356                     // output has been buffered it appears...)
 357                     tout.str(string());
 358 #define REPORT_FAIL_VG(M) do { \
 359     if (verbose) { \
 360         while (true) { \
 361             ssize_t c = read(vg_log_fd, buf, sizeof(buf)); \
 362             if (c == 0 || (c < 0 && errno != EINTR)) break; \
 363             if (c > 0) out << string(buf, c); \
 364         } \
 365     } \
 366     out << " " << col_red << M << col_reset; \
 367 } while (0)
 368                     // Record the current position so we can restore it so
 369                     // REPORT_FAIL_VG() gets the whole output.
 370                     off_t curpos = lseek(vg_log_fd, 0, SEEK_CUR);
 371                     char buf[4096];
 372                     while (true) {
 373                         ssize_t c = read(vg_log_fd, buf, sizeof(buf));
 374                         if (c == 0 || (c < 0 && errno != EINTR)) {
 375                             buf[0] = 0;
 376                             break;
 377                         }
 378                         if (c > 0) {
 379                             // Valgrind output has "==<pid>== \n" between
 380                             // report "records", so skip any lines like that,
 381                             // and also any warnings and continuation lines.
 382                             ssize_t i = 0;
 383                             while (true) {
 384                                 const char * spc;
 385                                 spc = static_cast<const char *>(
 386                                         memchr(buf + i, ' ', c - i));
 387                                 if (!spc) {
 388                                     i = c;
 389                                     break;
 390                                 }
 391                                 i = spc - buf;
 392                                 if (++i >= c) break;
 393                                 if (buf[i] == '\n')
 394                                     continue;
 395                                 if (c - i >= 8 &&
 396                                     (memcmp(buf + i, "Warning:", 8) == 0 ||
 397                                      memcmp(buf + i, "   ", 3) == 0)) {
 398                                     // Skip this line.
 399                                     i += 3;
 400                                     const char * nl;
 401                                     nl = static_cast<const char *>(
 402                                             memchr(buf + i, '\n', c - i));
 403                                     if (!nl) {
 404                                         i = c;
 405                                         break;
 406                                     }
 407                                     i = nl - buf;
 408                                     continue;
 409                                 }
 410                                 break;
 411                             }
 412
 413                             char *start = buf + i;
 414                             c -= i;
 415                             if (c > 128) c = 128;
 416
 417                             {
 418                                 const char *p;
 419                                 p = static_cast<const char*>(
 420                                         memchr(start, '\n', c));
 421                                 if (p != NULL) c = p - start;
 422                             }
 423
 424                             memmove(buf, start, c);
 425                             buf[c] = '\0';
 426                             break;
 427                         }
 428                     }
 429                     lseek(vg_log_fd, curpos, SEEK_SET);
 430
 431                     int vg_errs2 = VALGRIND_COUNT_ERRORS;
 432                     vg_errs = vg_errs2 - vg_errs;
 433                     VALGRIND_DO_LEAK_CHECK;
 434                     long vg_leaks2 = 0, vg_dubious2 = 0, vg_reachable2 = 0;
 435                     long dummy;
 436                     VALGRIND_COUNT_LEAKS(vg_leaks2, vg_dubious2, vg_reachable2,
 437                                          dummy);
 438                     (void)dummy;
 439                     vg_leaks = vg_leaks2 - vg_leaks;
 440                     vg_dubious = vg_dubious2 - vg_dubious;
 441                     vg_reachable = vg_reachable2 - vg_reachable;
 442                     if (vg_errs) {
 443                         string fail_msg(buf);
 444                         if (fail_msg.empty())
 445                             fail_msg = "VALGRIND DETECTED A PROBLEM";
 446                         REPORT_FAIL_VG(fail_msg);
 447                         return FAIL;
 448                     }
 449                     if (vg_leaks > 0) {
 450                         REPORT_FAIL_VG("LEAKED " << vg_leaks << " BYTES");
 451                         return FAIL;
 452                     }
 453                     if (vg_dubious > 0) {
 454                         // If code deliberately holds onto blocks by a pointer
 455                         // not to the start (e.g. languages/utilities.c does)
 456                         // then we need to rerun the test to see if the leak is
 457                         // real...
 458                         if (runcount == 0) {
 459                             out << col_yellow << " PROBABLY LEAKED MEMORY - RETRYING TEST" << col_reset;
 460                             ++runcount;
 461                             // Ensure that any cached memory from fd tracking
 462                             // is allocated before we rerun the test.
 463                             (void)fdtracker.check();
 464                             continue;
 465                         }
 466                         REPORT_FAIL_VG("PROBABLY LEAKED " << vg_dubious << " BYTES");
 467                         return FAIL;
 468                     }
 469                     if (vg_reachable > 0) {
 470                         // C++ STL implementations often "horde" released
 471                         // memory - for GCC 3.4 and newer the runtest script
 472                         // sets GLIBCXX_FORCE_NEW=1 which will disable this
 473                         // behaviour so we avoid this issue, but for older
 474                         // GCC and other compilers this may be an issue.
 475                         //
 476                         // See also:
 477                         // http://valgrind.org/docs/FAQ/#faq.reports
 478                         //
 479                         // For now, just use runcount to rerun the test and see
 480                         // if more is leaked - hopefully this shouldn't give
 481                         // false positives.
 482                         if (runcount == 0) {
 483                             out << col_yellow << " POSSIBLE UNRELEASED MEMORY - RETRYING TEST" << col_reset;
 484                             ++runcount;
 485                             // Ensure that any cached memory from fd tracking
 486                             // is allocated before we rerun the test.
 487                             (void)fdtracker.check();
 488                             continue;
 489                         }
 490                         REPORT_FAIL_VG("FAILED TO RELEASE " << vg_reachable << " BYTES");
 491                         return FAIL;
 492                     }
 493                 }
 494 #endif
 495                 if (!fdtracker.check()) {
 496                     if (runcount == 0) {
 497                         out << col_yellow << " POSSIBLE FDLEAK:" << fdtracker.get_message() << col_reset;
 498                         ++runcount;
 499                         continue;
 500                     }
 501                     out << col_red << " FDLEAK:" << fdtracker.get_message() << col_reset;
 502                     return FAIL;
 503                 }
 504             } catch (const TestFail &) {
 505                 out << ' ';
 506                 if (expected_failure) {
 507                     out << col_yellow << "XFAIL (" << expected_failure << ")";
 508                 } else {
 509                     out << col_red << "FAILED";
 510                 }
 511                 out << col_reset;
 512                 write_and_clear_tout();
 513                 return expected_failure ? XFAIL : FAIL;
 514             } catch (const TestSkip &) {
 515                 out << col_yellow << " SKIPPED" << col_reset;
 516                 write_and_clear_tout();
 517                 return SKIP;
 518 #ifndef NO_LIBXAPIAN
 519             } catch (const Xapian::Error &err) {
 520                 out << ' ';
 521                 string errclass = err.get_type();
 522                 if (expected_exception && expected_exception == errclass) {
 523                     out << col_yellow << "C++ FAILED TO CATCH " << errclass << col_reset;
 524                     return SKIP;
 525                 }
 526                 if (errclass == "NetworkError" &&
 527                     err.get_error_string() != NULL &&
 528                     strcmp(err.get_error_string(), strerror(ECHILD)) == 0) {
 529                     // ECHILD suggests we've run out of processes, and that's
 530                     // much more likely to be a system issue than a Xapian bug.
 531                     //
 532                     // We also see apparently spurious ECHILD on Debian
 533                     // buildds sometimes: https://bugs.debian.org/681941
 534                     out << col_yellow << "ECHILD in network code" << col_reset;
 535                     return SKIP;
 536                 }
 537
 538                 if (expected_failure) {
 539                     out << col_yellow << "XFAIL (" << expected_failure
 540                         << "): ";
 541                 } else {
 542                     out << col_red << "FAIL: ";
 543                 }
 544                 out << err.get_description() << col_reset;
 545                 write_and_clear_tout();
 546                 return expected_failure ? XFAIL : FAIL;
 547 #endif
 548             } catch (const string & msg) {
 549                 out << ' ';
 550                 if (expected_failure) {
 551                     out << col_yellow << "XFAIL (" << expected_failure
 552                         << "): ";
 553                 } else {
 554                     out << col_red << "FAIL: ";
 555                 }
 556                 out << "EXCEPTION std::string " << msg << col_reset;
 557                 write_and_clear_tout();
 558                 return expected_failure ? XFAIL : FAIL;
 559             } catch (const std::exception & e) {
 560                 out << ' ';
 561                 if (expected_failure) {
 562                     out << col_yellow << "XFAIL (" << expected_failure
 563                         << "): ";
 564                 } else {
 565                     out << col_red << "FAIL: ";
 566                 }
 567 #ifndef USE_RTTI
 568                 out << "std::exception";
 569 #else
 570                 const char * name = typeid(e).name();
 571 # ifdef HAVE_CXXABI_H
 572                 // __cxa_demangle() apparently requires GCC >= 3.1.
 573                 // Demangle the name which GCC returns for type_info::name().
 574                 int status;
 575                 char * realname = abi::__cxa_demangle(name, NULL, 0, &status);
 576                 if (realname) {
 577                     out << realname;
 578                     free(realname);
 579                 } else {
 580                     out << name;
 581                 }
 582 # else
 583                 out << name;
 584 # endif
 585 #endif
 586                 out << ": " << e.what() << col_reset;
 587                 write_and_clear_tout();
 588                 return expected_failure ? XFAIL : FAIL;
 589             } catch (const char * msg) {
 590                 out << ' ';
 591                 if (expected_failure) {
 592                     out << col_yellow << "XFAIL (" << expected_failure
 593                         << "): ";
 594                 } else {
 595                     out << col_red << "FAIL: ";
 596                 }
 597                 if (msg) {
 598                     out << "EXCEPTION char* " << msg;
 599                 } else {
 600                     out << "EXCEPTION (char*)NULL";
 601                 }
 602                 out << col_reset;
 603                 write_and_clear_tout();
 604                 return expected_failure ? XFAIL : FAIL;
 605             } catch (...) {
 606                 out << ' ';
 607                 if (expected_failure) {
 608                     out << col_yellow << "XFAIL (" << expected_failure
 609                         << "): ";
 610                 } else {
 611                     out << col_red << "FAIL: ";
 612                 }
 613                 out << "UNKNOWN EXCEPTION" << col_reset;
 614                 write_and_clear_tout();
 615                 return expected_failure ? XFAIL : FAIL;
 616             }
 617
 618             if (expected_failure) {
 619                 // Testcase marked as expected to fail but actually passed.
 620                 out << ' ' << col_red << "XPASS (" << expected_failure << ")"
 621                     << col_reset;
 622                 write_and_clear_tout();
 623                 return XPASS;
 624             }
 625             return PASS;
 626         }
 627
 628         // Caught a signal.
 629         const char *signame = "SIGNAL";
 630         bool show_addr = true;
 631         switch (signum) {
 632             case SIGSEGV: signame = "SIGSEGV"; break;
 633             case SIGFPE: signame = "SIGFPE"; break;
 634             case SIGILL: signame = "SIGILL"; break;
 635 #ifdef SIGBUS
 636             case SIGBUS: signame = "SIGBUS"; break;
 637 #endif
 638 #ifdef SIGSTKFLT
 639             case SIGSTKFLT:
 640                 signame = "SIGSTKFLT";
 641                 show_addr = false;
 642                 break;
 643 #endif
 644         }
 645         out << " " << col_red << signame;
 646         if (show_addr) {
 647             char buf[40];
 648             sprintf(buf, " at %p", sigaddr);
 649             out << buf;
 650         }
 651         out << col_reset;
 652         write_and_clear_tout();
 653         return FAIL;
 654     }
 655 }
 656
 657 test_driver::result
 658 test_driver::run_tests(vector<string>::const_iterator b,
 659                        vector<string>::const_iterator e)
 660 {
 661     return do_run_tests(b, e);
 662 }
 663
 664 test_driver::result
 665 test_driver::run_tests()
 666 {
 667     const vector<string> blank;
 668     return do_run_tests(blank.begin(), blank.end());
 669 }
 670
 671 test_driver::result
 672 test_driver::do_run_tests(vector<string>::const_iterator b,
 673                           vector<string>::const_iterator e)
 674 {
 675     set<string> m(b, e);
 676     bool check_name = !m.empty();
 677
 678     test_driver::result res;
 679
 680     for (const test_desc *test = tests; test->name; ++test) {
 681         bool do_this_test = !check_name;
 682         if (!do_this_test && m.find(test->name) != m.end())
 683             do_this_test = true;
 684         if (!do_this_test) {
 685             // if this test is "foo123" see if "foo" was listed
 686             // this way "./testprog foo" can run foo1, foo2, etc.
 687             string t = test->name;
 688             string::size_type i;
 689             i = t.find_last_not_of("0123456789") + 1;
 690             if (i != string::npos) {
 691                 t.resize(i);
 692                 if (m.find(t) != m.end()) do_this_test = true;
 693             }
 694         }
 695         if (do_this_test) {
 696             out << "Running test: " << test->name << "...";
 697             out.flush();
 698             test_driver::test_result test_res = runtest(test);
 699 #ifndef NO_LIBXAPIAN
 700             if (backendmanager)
 701                 backendmanager->clean_up();
 702 #endif
 703             switch (test_res) {
 704                 case PASS:
 705                     ++res.succeeded;
 706                     if (verbose || !use_cr) {
 707                         out << col_green << " ok" << col_reset << endl;
 708                     } else {
 709                         out << "\r                                                                               \r";
 710                     }
 711                     break;
 712                 case XFAIL:
 713                     ++res.xfailed;
 714                     out << endl;
 715                     break;
 716                 case FAIL:
 717                     ++res.failed;
 718                     out << endl;
 719                     if (abort_on_error) {
 720                         throw "Test failed - aborting further tests";
 721                     }
 722                     break;
 723                 case XPASS:
 724                     ++res.xpassed;
 725                     out << endl;
 726                     if (abort_on_error) {
 727                         throw "Test marked as XFAIL passed - aborting further tests";
 728                     }
 729                     break;
 730                 case SKIP:
 731                     ++res.skipped;
 732                     out << endl;
 733                     // ignore the result of this test.
 734                     break;
 735             }
 736         }
 737     }
 738     return res;
 739 }
 740
 741 void
 742 test_driver::usage()
 743 {
 744     cout << "Usage: " << argv0 << " [-v|--verbose] [-o|--abort-on-error] " << opt_help
 745          << "[TESTNAME]..." << endl;
 746     cout << "       " << argv0 << " [-h|--help]" << endl;
 747     exit(1);
 748 }
 749
 750 /* Needs C linkage so we can pass it to atexit() without problems. */
 751 extern "C" {
 752 // Call upon program exit if there's more than one test run.
 753 static void
 754 report_totals(void)
 755 {
 756     test_driver::report(test_driver::total, "total");
 757 }
 758 }
 759
 760 void
 761 test_driver::report(const test_driver::result &r, const string &desc)
 762 {
 763     // Report totals at the end if we reported two or more subtotals.
 764     if (++runs == 2) atexit(report_totals);
 765
 766     if (r.succeeded != 0 || r.failed != 0) {
 767         cout << argv0 << " " << desc << ": ";
 768
 769         if (r.failed == 0 && r.xpassed == 0)
 770             cout << "All ";
 771
 772         cout << col_green << r.succeeded << col_reset << " tests passed";
 773
 774         if (r.failed != 0)
 775             cout << ", " << col_red << r.failed << col_reset << " failed";
 776
 777         if (r.xpassed != 0)
 778             cout << ", " << col_red << r.xpassed << col_reset
 779                  << " expected failures passed";
 780
 781         if (r.xfailed != 0)
 782             cout << ", " << col_yellow << r.xfailed << col_reset
 783                  << " expected failures";
 784
 785         if (r.skipped) {
 786             cout << ", " << col_yellow << r.skipped << col_reset
 787                  << " skipped." << endl;
 788         } else {
 789             cout << "." << endl;
 790         }
 791     }
 792 }
 793
 794 void
 795 test_driver::add_command_line_option(const string &l, char s, string * arg)
 796 {
 797     short_opts.insert(make_pair(int(s), arg));
 798     opt_help += "[-";
 799     opt_help += s;
 800     opt_help += ' ';
 801     opt_help += l;
 802     opt_help += "] ";
 803 }
 804
 805 void
 806 test_driver::parse_command_line(int argc, char **argv)
 807 {
 808     argv0 = argv[0];
 809
 810 #ifdef HAVE_VALGRIND
 811     if (RUNNING_ON_VALGRIND) {
 812         if (getenv("XAPIAN_TESTSUITE_VALGRIND") != NULL) {
 813             // Open the valgrind log file, and unlink it.
 814             char fname[64];
 815             sprintf(fname, ".valgrind.log.%lu",
 816                     static_cast<unsigned long>(getpid()));
 817             vg_log_fd = open(fname, O_RDONLY|O_NONBLOCK|O_CLOEXEC);
 818             if (vg_log_fd != -1) unlink(fname);
 819         }
 820     }
 821 #endif
 822
 823 #ifndef __WIN32__
 824     {
 825         bool colourise = true;
 826         const char *p = getenv("XAPIAN_TESTSUITE_OUTPUT");
 827         if (p == NULL || !*p || strcmp(p, "auto") == 0) {
 828             colourise = isatty(1);
 829         } else if (strcmp(p, "plain") == 0) {
 830             colourise = false;
 831         }
 832         if (colourise) {
 833             col_red = "\x1b[1m\x1b[31m";
 834             col_green = "\x1b[1m\x1b[32m";
 835             col_yellow = "\x1b[1m\x1b[33m";
 836             col_reset = "\x1b[0m";
 837             use_cr = true;
 838         }
 839     }
 840 #endif
 841
 842     static const struct option long_opts[] = {
 843         {"verbose",             no_argument, 0, 'v'},
 844         {"abort-on-error",      no_argument, 0, 'o'},
 845         {"help",                no_argument, 0, 'h'},
 846         {NULL,                  0, 0, 0}
 847     };
 848
 849     string short_opts_string = "voh";
 850     map<int, string *>::const_iterator i;
 851     for (i = short_opts.begin(); i != short_opts.end(); ++i) {
 852         short_opts_string += char(i->first);
 853         short_opts_string += ':';
 854     }
 855     const char * opts = short_opts_string.c_str();
 856
 857     int c;
 858     while ((c = gnu_getopt_long(argc, argv, opts, long_opts, 0)) != -1) {
 859         switch (c) {
 860             case 'v':
 861                 ++verbose;
 862                 break;
 863             case 'o':
 864                 abort_on_error = true;
 865                 break;
 866             default: {
 867                 i = short_opts.find(c);
 868                 if (i != short_opts.end()) {
 869                     i->second->assign(optarg);
 870                     break;
 871                 }
 872                 // -h or unrecognised option
 873                 usage();
 874                 return; // usage() doesn't return ...
 875             }
 876         }
 877     }
 878
 879     if (verbose == 0) {
 880         const char *p = getenv("VERBOSE");
 881         if (p != NULL) {
 882             verbose = atoi(p);
 883         }
 884     }
 885
 886     while (argv[optind]) {
 887         test_names.push_back(string(argv[optind]));
 888         optind++;
 889     }
 890 }
 891
 892 int
 893 test_driver::run(const test_desc *tests)
 894 {
 895     test_driver driver(tests);
 896
 897     test_driver::result myresult;
 898     myresult = driver.run_tests(test_names.begin(), test_names.end());
 899
 900     subtotal += myresult;
 901
 902     // Return value is a Unix-style exit code, so 0 for success and 1 for
 903     // failure.
 904     return myresult.failed > 0 || myresult.xpassed > 0;
 905 }
 906
 907 bool
 908 TEST_EQUAL_DOUBLE_(double a, double b)
 909 {
 910     if (a == b) return true;
 911     return (ceil(log10(max(fabs(a), fabs(b)))) - log10(fabs(a - b)) > DBL_DIG);
 912 }