xapian-core/tests/harness/testsuite.cc

   1 /* testsuite.cc: a test suite engine
   2  *
   3  * Copyright 1999,2000,2001 BrightStation PLC
   4  * Copyright 2002 Ananova Ltd
   5  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2012,2013,2015,2016,2017 Olly Betts
   6  * Copyright 2007 Richard Boulton
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License as
  10  * published by the Free Software Foundation; either version 2 of the
  11  * License, or (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16  * GNU General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * along with this program; if not, write to the Free Software
  20  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  21  * USA
  22  */
  23
  24 #include <config.h>
  25
  26 #include "testsuite.h"
  27
  28 #ifndef NO_LIBXAPIAN
  29 # include "backendmanager.h"
  30 #endif
  31 #include "fdtracker.h"
  32 #include "testrunner.h"
  33 #include "safeerrno.h"
  34 #include "safeunistd.h"
  35
  36 #ifdef HAVE_VALGRIND
  37 # include <valgrind/memcheck.h>
  38 # include <sys/types.h>
  39 # include "safefcntl.h"
  40 #endif
  41
  42 #include <algorithm>
  43 #include <iomanip>
  44 #include <iostream>
  45 #include <set>
  46
  47 #include <cfloat> // For DBL_DIG.
  48 #include <cmath> // For ceil, fabs, log10.
  49 #include <cstdio>
  50 #include <cstdlib>
  51 #include <cstring>
  52
  53 #include "gnu_getopt.h"
  54
  55 #include <setjmp.h>
  56 #include <signal.h>
  57
  58 #include <exception>
  59 #ifdef USE_RTTI
  60 # include <typeinfo>
  61 # ifdef HAVE_CXXABI_H
  62 #  include <cxxabi.h>
  63 # endif
  64 #endif
  65
  66 #ifndef NO_LIBXAPIAN
  67 # include <xapian/error.h>
  68 #endif
  69 #include "filetests.h"
  70 #include "noreturn.h"
  71 #include "stringutils.h"
  72
  73 using namespace std;
  74
  75 /// The global verbose flag.
  76 int verbose;
  77
  78 #ifdef HAVE_VALGRIND
  79 static int vg_log_fd = -1;
  80 #endif
  81
  82 #ifdef HAVE_SIGSETJMP
  83 # define SIGSETJMP(ENV, SAVESIGS) sigsetjmp(ENV, SAVESIGS)
  84 # define SIGLONGJMP(ENV, VAL) siglongjmp(ENV, VAL)
  85 # define SIGJMP_BUF sigjmp_buf
  86 #else
  87 # define SIGSETJMP(ENV, SAVESIGS) setjmp(ENV)
  88 # define SIGLONGJMP(ENV, VAL) longjmp(ENV, VAL)
  89 # define SIGJMP_BUF jmp_buf
  90 #endif
  91
  92 /// The exception type we were expecting in TEST_EXCEPTION
  93 //  We use this to attempt to diagnose when the code fails to catch an
  94 //  exception when it should (due to a compiler or runtime fault in
  95 //  GCC 2.95 it seems)
  96 const char * expected_exception = NULL;
  97
  98 const char* expected_failure;
  99
 100 /// The debug printing stream
 101 std::ostringstream tout;
 102
 103 int test_driver::runs = 0;
 104 test_driver::result test_driver::subtotal;
 105 test_driver::result test_driver::total;
 106 string test_driver::argv0;
 107 string test_driver::opt_help;
 108 map<int, string *> test_driver::short_opts;
 109 vector<string> test_driver::test_names;
 110 bool test_driver::abort_on_error = false;
 111 string test_driver::col_red, test_driver::col_green;
 112 string test_driver::col_yellow, test_driver::col_reset;
 113 bool test_driver::use_cr = false;
 114
 115 void
 116 test_driver::write_and_clear_tout()
 117 {
 118     const string & s = tout.str();
 119     if (!s.empty()) {
 120         out << '\n' << s;
 121         tout.str(string());
 122     }
 123 }
 124
 125 string
 126 test_driver::get_srcdir()
 127 {
 128     char *p = getenv("srcdir");
 129     if (p != NULL) return string(p);
 130
 131     // Default srcdir to the pathname of argv[0].
 132     string srcdir(argv0);
 133     string::size_type i = srcdir.find_last_of(DIR_SEPS);
 134     string srcfile;
 135     if (i != string::npos) {
 136         srcfile.assign(srcdir, i + 1, string::npos);
 137         srcdir.erase(i);
 138         // libtool may put the real executable in .libs.
 139         i = srcdir.find_last_of(DIR_SEPS);
 140         if (srcdir.substr(i + 1) == ".libs") {
 141             srcdir.erase(i);
 142             // And it may have an "lt-" prefix.
 143             if (startswith(srcfile, "lt-")) srcfile.erase(0, 3);
 144         }
 145     } else {
 146         // No path of argv[0], so default srcdir to the current directory.
 147         // This may not work if libtool is involved as the true executable is
 148         // sometimes in ".libs".
 149         srcfile = srcdir;
 150         srcdir = ".";
 151     }
 152
 153     // Remove any trailing ".exe" suffix, since some platforms add this.
 154     if (endswith(srcfile, ".exe")) srcfile.resize(srcfile.size() - 4);
 155
 156     // Sanity check.
 157     if (!file_exists(srcdir + '/' + srcfile + ".cc")) {
 158         cout << argv0
 159              << ": srcdir is not in the environment and I can't guess it!\n"
 160                 "Run test programs using the runtest script - see HACKING for details"
 161              << endl;
 162         exit(1);
 163     }
 164     return srcdir;
 165 }
 166
 167 test_driver::test_driver(const test_desc *tests_)
 168         : out(cout.rdbuf()), tests(tests_)
 169 {
 170     tout << boolalpha;
 171 }
 172
 173 static SIGJMP_BUF jb;
 174 static int signum = 0;
 175 static void * sigaddr = NULL;
 176
 177 // Needs C linkage so we can pass it to sigaction()/signal() without problems.
 178 extern "C" {
 179
 180 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 181 XAPIAN_NORETURN(static void handle_sig(int signum_, siginfo_t *si, void *));
 182 static void handle_sig(int signum_, siginfo_t *si, void *)
 183 {
 184     // Disable all our signal handlers to avoid problems if the signal
 185     // handling code causes a signal.
 186     struct sigaction sa;
 187     sa.sa_handler = SIG_DFL;
 188     sigemptyset(&sa.sa_mask);
 189     sa.sa_flags = 0;
 190     // We set the handlers with SA_RESETHAND, but that will only reset the
 191     // handler for the signal which fired.
 192     if (signum_ != SIGSEGV) sigaction(SIGSEGV, &sa, NULL);
 193     if (signum_ != SIGFPE) sigaction(SIGFPE, &sa, NULL);
 194     if (signum_ != SIGILL) sigaction(SIGILL, &sa, NULL);
 195 # ifdef SIGBUS
 196     if (signum_ != SIGBUS) sigaction(SIGBUS, &sa, NULL);
 197 # endif
 198 # ifdef SIGSTKFLT
 199     if (signum_ != SIGSTKFLT) sigaction(SIGSTKFLT, &sa, NULL);
 200 # endif
 201     signum = signum_;
 202     sigaddr = si->si_addr;
 203     SIGLONGJMP(jb, 1);
 204 }
 205
 206 #else
 207
 208 XAPIAN_NORETURN(static void handle_sig(int signum_));
 209 static void handle_sig(int signum_)
 210 {
 211     // Disable all our signal handlers to avoid problems if the signal
 212     // handling code causes a signal.
 213     signal(SIGSEGV, SIG_DFL);
 214     signal(SIGFPE, SIG_DFL);
 215     signal(SIGILL, SIG_DFL);
 216 #ifdef SIGBUS
 217     signal(SIGBUS, SIG_DFL);
 218 #endif
 219 #ifdef SIGSTKFLT
 220     signal(SIGSTKFLT, SIG_DFL);
 221 #endif
 222     signum = signum_;
 223     SIGLONGJMP(jb, 1);
 224 }
 225 #endif
 226
 227 }
 228
 229 class SignalRedirector {
 230   private:
 231     bool active;
 232   public:
 233     SignalRedirector() : active(false) { }
 234     void activate() {
 235         active = true;
 236         signum = 0;
 237         sigaddr = NULL;
 238         // SA_SIGINFO not universal (e.g. not present on Linux < 2.2 and Hurd).
 239 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 240         struct sigaction sa;
 241         sa.sa_sigaction = handle_sig;
 242         sigemptyset(&sa.sa_mask);
 243         sa.sa_flags = SA_RESETHAND|SA_SIGINFO;
 244         sigaction(SIGSEGV, &sa, NULL);
 245         sigaction(SIGFPE, &sa, NULL);
 246         sigaction(SIGILL, &sa, NULL);
 247 # ifdef SIGBUS
 248         sigaction(SIGBUS, &sa, NULL);
 249 # endif
 250 # ifdef SIGSTKFLT
 251         sigaction(SIGSTKFLT, &sa, NULL);
 252 # endif
 253 #else
 254         signal(SIGSEGV, handle_sig);
 255         signal(SIGFPE, handle_sig);
 256         signal(SIGILL, handle_sig);
 257 # ifdef SIGBUS
 258         signal(SIGBUS, handle_sig);
 259 # endif
 260 # ifdef SIGSTKFLT
 261         signal(SIGSTKFLT, handle_sig);
 262 # endif
 263 #endif
 264     }
 265     ~SignalRedirector() {
 266         if (active) {
 267 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 268             struct sigaction sa;
 269             sa.sa_handler = SIG_DFL;
 270             sigemptyset(&sa.sa_mask);
 271             sa.sa_flags = 0;
 272             sigaction(SIGSEGV, &sa, NULL);
 273             sigaction(SIGFPE, &sa, NULL);
 274             sigaction(SIGILL, &sa, NULL);
 275 # ifdef SIGBUS
 276             sigaction(SIGBUS, &sa, NULL);
 277 # endif
 278 # ifdef SIGSTKFLT
 279             sigaction(SIGSTKFLT, &sa, NULL);
 280 # endif
 281 #else
 282             signal(SIGSEGV, SIG_DFL);
 283             signal(SIGFPE, SIG_DFL);
 284             signal(SIGILL, SIG_DFL);
 285 # ifdef SIGBUS
 286             signal(SIGBUS, SIG_DFL);
 287 # endif
 288 # ifdef SIGSTKFLT
 289             signal(SIGSTKFLT, SIG_DFL);
 290 # endif
 291 #endif
 292         }
 293     }
 294 };
 295
 296 //  A wrapper around the tests to trap exceptions,
 297 //  and avoid having to catch them in every test function.
 298 //  If this test driver is used for anything other than
 299 //  Xapian tests, then this ought to be provided by
 300 //  the client, really.
 301 //  return: test_driver::PASS, test_driver::FAIL, test_driver::SKIP,
 302 //  test_driver::XFAIL or test_driver:XPASS.
 303 test_driver::test_result
 304 test_driver::runtest(const test_desc *test)
 305 {
 306     // This is used to make a note of how many times we've run the test
 307     volatile int runcount = 0;
 308
 309     FDTracker fdtracker;
 310     fdtracker.init();
 311
 312     while (true) {
 313         tout.str(string());
 314         if (SIGSETJMP(jb, 1) == 0) {
 315             SignalRedirector sig; // use object so signal handlers are reset
 316             static bool catch_signals =
 317                 (getenv("XAPIAN_TESTSUITE_SIG_DFL") == NULL);
 318             if (catch_signals) sig.activate();
 319             try {
 320                 expected_exception = NULL;
 321                 expected_failure = NULL;
 322 #ifdef HAVE_VALGRIND
 323                 int vg_errs = 0;
 324                 long vg_leaks = 0, vg_dubious = 0, vg_reachable = 0;
 325                 if (vg_log_fd != -1) {
 326                     VALGRIND_DO_LEAK_CHECK;
 327                     vg_errs = VALGRIND_COUNT_ERRORS;
 328                     long dummy;
 329                     VALGRIND_COUNT_LEAKS(vg_leaks, vg_dubious, vg_reachable, dummy);
 330                     (void)dummy;
 331                     // Skip past any unread log output.
 332                     lseek(vg_log_fd, 0, SEEK_END);
 333                 }
 334 #endif
 335                 if (!test->run()) {
 336                     out << ' ';
 337                     if (expected_failure) {
 338                         out << col_yellow << "XFAIL (" << expected_failure << ")";
 339                     } else {
 340                         out << col_red << "FAILED";
 341                     }
 342                     out << col_reset;
 343                     write_and_clear_tout();
 344                     return expected_failure ? XFAIL : FAIL;
 345                 }
 346                 if (verbose > 1)
 347                     write_and_clear_tout();
 348 #ifndef NO_LIBXAPIAN
 349                 if (backendmanager)
 350                     backendmanager->clean_up();
 351 #endif
 352 #ifdef HAVE_VALGRIND
 353                 if (vg_log_fd != -1) {
 354                     // We must empty tout before asking valgrind to perform its
 355                     // leak checks, otherwise the buffers holding the output
 356                     // may be identified as a memory leak (especially if >1K of
 357                     // output has been buffered it appears...)
 358                     tout.str(string());
 359 #define REPORT_FAIL_VG(M) do { \
 360     if (verbose) { \
 361         while (true) { \
 362             ssize_t c = read(vg_log_fd, buf, sizeof(buf)); \
 363             if (c == 0 || (c < 0 && errno != EINTR)) break; \
 364             if (c > 0) out << string(buf, c); \
 365         } \
 366     } \
 367     out << " " << col_red << M << col_reset; \
 368 } while (0)
 369                     // Record the current position so we can restore it so
 370                     // REPORT_FAIL_VG() gets the whole output.
 371                     off_t curpos = lseek(vg_log_fd, 0, SEEK_CUR);
 372                     char buf[4096];
 373                     while (true) {
 374                         ssize_t c = read(vg_log_fd, buf, sizeof(buf));
 375                         if (c == 0 || (c < 0 && errno != EINTR)) {
 376                             buf[0] = 0;
 377                             break;
 378                         }
 379                         if (c > 0) {
 380                             // Valgrind output has "==<pid>== \n" between
 381                             // report "records", so skip any lines like that,
 382                             // and also any warnings and continuation lines.
 383                             ssize_t i = 0;
 384                             while (true) {
 385                                 const char * spc;
 386                                 spc = static_cast<const char *>(
 387                                         memchr(buf + i, ' ', c - i));
 388                                 if (!spc) {
 389                                     i = c;
 390                                     break;
 391                                 }
 392                                 i = spc - buf;
 393                                 if (++i >= c) break;
 394                                 if (buf[i] == '\n')
 395                                     continue;
 396                                 if (c - i >= 8 &&
 397                                     (memcmp(buf + i, "Warning:", 8) == 0 ||
 398                                      memcmp(buf + i, "   ", 3) == 0)) {
 399                                     // Skip this line.
 400                                     i += 3;
 401                                     const char * nl;
 402                                     nl = static_cast<const char *>(
 403                                             memchr(buf + i, '\n', c - i));
 404                                     if (!nl) {
 405                                         i = c;
 406                                         break;
 407                                     }
 408                                     i = nl - buf;
 409                                     continue;
 410                                 }
 411                                 break;
 412                             }
 413
 414                             char *start = buf + i;
 415                             c -= i;
 416                             if (c > 128) c = 128;
 417
 418                             {
 419                                 const char *p;
 420                                 p = static_cast<const char*>(
 421                                         memchr(start, '\n', c));
 422                                 if (p != NULL) c = p - start;
 423                             }
 424
 425                             memmove(buf, start, c);
 426                             buf[c] = '\0';
 427                             break;
 428                         }
 429                     }
 430                     lseek(vg_log_fd, curpos, SEEK_SET);
 431
 432                     int vg_errs2 = VALGRIND_COUNT_ERRORS;
 433                     vg_errs = vg_errs2 - vg_errs;
 434                     VALGRIND_DO_LEAK_CHECK;
 435                     long vg_leaks2 = 0, vg_dubious2 = 0, vg_reachable2 = 0;
 436                     long dummy;
 437                     VALGRIND_COUNT_LEAKS(vg_leaks2, vg_dubious2, vg_reachable2,
 438                                          dummy);
 439                     (void)dummy;
 440                     vg_leaks = vg_leaks2 - vg_leaks;
 441                     vg_dubious = vg_dubious2 - vg_dubious;
 442                     vg_reachable = vg_reachable2 - vg_reachable;
 443                     if (vg_errs) {
 444                         string fail_msg(buf);
 445                         if (fail_msg.empty())
 446                             fail_msg = "VALGRIND DETECTED A PROBLEM";
 447                         REPORT_FAIL_VG(fail_msg);
 448                         return FAIL;
 449                     }
 450                     if (vg_leaks > 0) {
 451                         REPORT_FAIL_VG("LEAKED " << vg_leaks << " BYTES");
 452                         return FAIL;
 453                     }
 454                     if (vg_dubious > 0) {
 455                         // If code deliberately holds onto blocks by a pointer
 456                         // not to the start (e.g. languages/utilities.c does)
 457                         // then we need to rerun the test to see if the leak is
 458                         // real...
 459                         if (runcount == 0) {
 460                             out << col_yellow << " PROBABLY LEAKED MEMORY - RETRYING TEST" << col_reset;
 461                             ++runcount;
 462                             // Ensure that any cached memory from fd tracking
 463                             // is allocated before we rerun the test.
 464                             (void)fdtracker.check();
 465                             continue;
 466                         }
 467                         REPORT_FAIL_VG("PROBABLY LEAKED " << vg_dubious << " BYTES");
 468                         return FAIL;
 469                     }
 470                     if (vg_reachable > 0) {
 471                         // C++ STL implementations often "horde" released
 472                         // memory - for GCC 3.4 and newer the runtest script
 473                         // sets GLIBCXX_FORCE_NEW=1 which will disable this
 474                         // behaviour so we avoid this issue, but for older
 475                         // GCC and other compilers this may be an issue.
 476                         //
 477                         // See also:
 478                         // http://valgrind.org/docs/FAQ/#faq.reports
 479                         //
 480                         // For now, just use runcount to rerun the test and see
 481                         // if more is leaked - hopefully this shouldn't give
 482                         // false positives.
 483                         if (runcount == 0) {
 484                             out << col_yellow << " POSSIBLE UNRELEASED MEMORY - RETRYING TEST" << col_reset;
 485                             ++runcount;
 486                             // Ensure that any cached memory from fd tracking
 487                             // is allocated before we rerun the test.
 488                             (void)fdtracker.check();
 489                             continue;
 490                         }
 491                         REPORT_FAIL_VG("FAILED TO RELEASE " << vg_reachable << " BYTES");
 492                         return FAIL;
 493                     }
 494                 }
 495 #endif
 496                 if (!fdtracker.check()) {
 497                     if (runcount == 0) {
 498                         out << col_yellow << " POSSIBLE FDLEAK:" << fdtracker.get_message() << col_reset;
 499                         ++runcount;
 500                         continue;
 501                     }
 502                     out << col_red << " FDLEAK:" << fdtracker.get_message() << col_reset;
 503                     return FAIL;
 504                 }
 505             } catch (const TestFail &) {
 506                 out << ' ';
 507                 if (expected_failure) {
 508                     out << col_yellow << "XFAIL (" << expected_failure << ")";
 509                 } else {
 510                     out << col_red << "FAILED";
 511                 }
 512                 out << col_reset;
 513                 write_and_clear_tout();
 514                 return expected_failure ? XFAIL : FAIL;
 515             } catch (const TestSkip &) {
 516                 out << col_yellow << " SKIPPED" << col_reset;
 517                 write_and_clear_tout();
 518                 return SKIP;
 519 #ifndef NO_LIBXAPIAN
 520             } catch (const Xapian::Error &err) {
 521                 out << ' ';
 522                 string errclass = err.get_type();
 523                 if (expected_exception && expected_exception == errclass) {
 524                     out << col_yellow << "C++ FAILED TO CATCH " << errclass << col_reset;
 525                     return SKIP;
 526                 }
 527                 if (errclass == "NetworkError" &&
 528                     err.get_error_string() != NULL &&
 529                     strcmp(err.get_error_string(), strerror(ECHILD)) == 0) {
 530                     // ECHILD suggests we've run out of processes, and that's
 531                     // much more likely to be a system issue than a Xapian bug.
 532                     //
 533                     // We also see apparently spurious ECHILD on Debian
 534                     // buildds sometimes: https://bugs.debian.org/681941
 535                     out << col_yellow << "ECHILD in network code" << col_reset;
 536                     return SKIP;
 537                 }
 538
 539                 if (expected_failure) {
 540                     out << col_yellow << "XFAIL (" << expected_failure
 541                         << "): ";
 542                 } else {
 543                     out << col_red << "FAIL: ";
 544                 }
 545                 out << err.get_description() << col_reset;
 546                 write_and_clear_tout();
 547                 return expected_failure ? XFAIL : FAIL;
 548 #endif
 549             } catch (const string & msg) {
 550                 out << ' ';
 551                 if (expected_failure) {
 552                     out << col_yellow << "XFAIL (" << expected_failure
 553                         << "): ";
 554                 } else {
 555                     out << col_red << "FAIL: ";
 556                 }
 557                 out << "EXCEPTION std::string " << msg << col_reset;
 558                 write_and_clear_tout();
 559                 return expected_failure ? XFAIL : FAIL;
 560             } catch (const std::exception & e) {
 561                 out << ' ';
 562                 if (expected_failure) {
 563                     out << col_yellow << "XFAIL (" << expected_failure
 564                         << "): ";
 565                 } else {
 566                     out << col_red << "FAIL: ";
 567                 }
 568 #ifndef USE_RTTI
 569                 out << "std::exception";
 570 #else
 571                 const char * name = typeid(e).name();
 572 # ifdef HAVE_CXXABI_H
 573                 // __cxa_demangle() apparently requires GCC >= 3.1.
 574                 // Demangle the name which GCC returns for type_info::name().
 575                 int status;
 576                 char * realname = abi::__cxa_demangle(name, NULL, 0, &status);
 577                 if (realname) {
 578                     out << realname;
 579                     free(realname);
 580                 } else {
 581                     out << name;
 582                 }
 583 # else
 584                 out << name;
 585 # endif
 586 #endif
 587                 out << ": " << e.what() << col_reset;
 588                 write_and_clear_tout();
 589                 return expected_failure ? XFAIL : FAIL;
 590             } catch (const char * msg) {
 591                 out << ' ';
 592                 if (expected_failure) {
 593                     out << col_yellow << "XFAIL (" << expected_failure
 594                         << "): ";
 595                 } else {
 596                     out << col_red << "FAIL: ";
 597                 }
 598                 if (msg) {
 599                     out << "EXCEPTION char* " << msg;
 600                 } else {
 601                     out << "EXCEPTION (char*)NULL";
 602                 }
 603                 out << col_reset;
 604                 write_and_clear_tout();
 605                 return expected_failure ? XFAIL : FAIL;
 606             } catch (...) {
 607                 out << ' ';
 608                 if (expected_failure) {
 609                     out << col_yellow << "XFAIL (" << expected_failure
 610                         << "): ";
 611                 } else {
 612                     out << col_red << "FAIL: ";
 613                 }
 614                 out << "UNKNOWN EXCEPTION" << col_reset;
 615                 write_and_clear_tout();
 616                 return expected_failure ? XFAIL : FAIL;
 617             }
 618
 619             if (expected_failure) {
 620                 // Testcase marked as expected to fail but actually passed.
 621                 out << ' ' << col_red << "XPASS (" << expected_failure << ")"
 622                     << col_reset;
 623                 write_and_clear_tout();
 624                 return XPASS;
 625             }
 626             return PASS;
 627         }
 628
 629         // Caught a signal.
 630         const char *signame = "SIGNAL";
 631         bool show_addr = true;
 632         switch (signum) {
 633             case SIGSEGV: signame = "SIGSEGV"; break;
 634             case SIGFPE: signame = "SIGFPE"; break;
 635             case SIGILL: signame = "SIGILL"; break;
 636 #ifdef SIGBUS
 637             case SIGBUS: signame = "SIGBUS"; break;
 638 #endif
 639 #ifdef SIGSTKFLT
 640             case SIGSTKFLT:
 641                 signame = "SIGSTKFLT";
 642                 show_addr = false;
 643                 break;
 644 #endif
 645         }
 646         out << " " << col_red << signame;
 647         if (show_addr) {
 648             char buf[40];
 649             sprintf(buf, " at %p", sigaddr);
 650             out << buf;
 651         }
 652         out << col_reset;
 653         write_and_clear_tout();
 654         return FAIL;
 655     }
 656 }
 657
 658 test_driver::result
 659 test_driver::run_tests(vector<string>::const_iterator b,
 660                        vector<string>::const_iterator e)
 661 {
 662     return do_run_tests(b, e);
 663 }
 664
 665 test_driver::result
 666 test_driver::run_tests()
 667 {
 668     const vector<string> blank;
 669     return do_run_tests(blank.begin(), blank.end());
 670 }
 671
 672 test_driver::result
 673 test_driver::do_run_tests(vector<string>::const_iterator b,
 674                           vector<string>::const_iterator e)
 675 {
 676     set<string> m(b, e);
 677     bool check_name = !m.empty();
 678
 679     test_driver::result res;
 680
 681     for (const test_desc *test = tests; test->name; ++test) {
 682         bool do_this_test = !check_name;
 683         if (!do_this_test && m.find(test->name) != m.end())
 684             do_this_test = true;
 685         if (!do_this_test) {
 686             // if this test is "foo123" see if "foo" was listed
 687             // this way "./testprog foo" can run foo1, foo2, etc.
 688             string t = test->name;
 689             string::size_type i;
 690             i = t.find_last_not_of("0123456789") + 1;
 691             if (i != string::npos) {
 692                 t.resize(i);
 693                 if (m.find(t) != m.end()) do_this_test = true;
 694             }
 695         }
 696         if (do_this_test) {
 697             out << "Running test: " << test->name << "...";
 698             out.flush();
 699             test_driver::test_result test_res = runtest(test);
 700 #ifndef NO_LIBXAPIAN
 701             if (backendmanager)
 702                 backendmanager->clean_up();
 703 #endif
 704             switch (test_res) {
 705                 case PASS:
 706                     ++res.succeeded;
 707                     if (verbose || !use_cr) {
 708                         out << col_green << " ok" << col_reset << endl;
 709                     } else {
 710                         out << "\r                                                                               \r";
 711                     }
 712                     break;
 713                 case XFAIL:
 714                     ++res.xfailed;
 715                     out << endl;
 716                     break;
 717                 case FAIL:
 718                     ++res.failed;
 719                     out << endl;
 720                     if (abort_on_error) {
 721                         throw "Test failed - aborting further tests";
 722                     }
 723                     break;
 724                 case XPASS:
 725                     ++res.xpassed;
 726                     out << endl;
 727                     if (abort_on_error) {
 728                         throw "Test marked as XFAIL passed - aborting further tests";
 729                     }
 730                     break;
 731                 case SKIP:
 732                     ++res.skipped;
 733                     out << endl;
 734                     // ignore the result of this test.
 735                     break;
 736             }
 737         }
 738     }
 739     return res;
 740 }
 741
 742 void
 743 test_driver::usage()
 744 {
 745     cout << "Usage: " << argv0 << " [-v|--verbose] [-o|--abort-on-error] " << opt_help
 746          << "[TESTNAME]..." << endl;
 747     cout << "       " << argv0 << " [-h|--help]" << endl;
 748     exit(1);
 749 }
 750
 751 /* Needs C linkage so we can pass it to atexit() without problems. */
 752 extern "C" {
 753 // Call upon program exit if there's more than one test run.
 754 static void
 755 report_totals(void)
 756 {
 757     test_driver::report(test_driver::total, "total");
 758 }
 759 }
 760
 761 void
 762 test_driver::report(const test_driver::result &r, const string &desc)
 763 {
 764     // Report totals at the end if we reported two or more subtotals.
 765     if (++runs == 2) atexit(report_totals);
 766
 767     if (r.succeeded != 0 || r.failed != 0) {
 768         cout << argv0 << " " << desc << ": ";
 769
 770         if (r.failed == 0 && r.xpassed == 0)
 771             cout << "All ";
 772
 773         cout << col_green << r.succeeded << col_reset << " tests passed";
 774
 775         if (r.failed != 0)
 776             cout << ", " << col_red << r.failed << col_reset << " failed";
 777
 778         if (r.xpassed != 0)
 779             cout << ", " << col_red << r.xpassed << col_reset
 780                  << " expected failures passed";
 781
 782         if (r.xfailed != 0)
 783             cout << ", " << col_yellow << r.xfailed << col_reset
 784                  << " expected failures";
 785
 786         if (r.skipped) {
 787             cout << ", " << col_yellow << r.skipped << col_reset
 788                  << " skipped." << endl;
 789         } else {
 790             cout << "." << endl;
 791         }
 792     }
 793 }
 794
 795 void
 796 test_driver::add_command_line_option(const string &l, char s, string * arg)
 797 {
 798     short_opts.insert(make_pair(int(s), arg));
 799     opt_help += "[-";
 800     opt_help += s;
 801     opt_help += ' ';
 802     opt_help += l;
 803     opt_help += "] ";
 804 }
 805
 806 void
 807 test_driver::parse_command_line(int argc, char **argv)
 808 {
 809     argv0 = argv[0];
 810
 811 #ifdef HAVE_VALGRIND
 812     if (RUNNING_ON_VALGRIND) {
 813         if (getenv("XAPIAN_TESTSUITE_VALGRIND") != NULL) {
 814             // Open the valgrind log file, and unlink it.
 815             char fname[64];
 816             sprintf(fname, ".valgrind.log.%lu",
 817                     static_cast<unsigned long>(getpid()));
 818             vg_log_fd = open(fname, O_RDONLY|O_NONBLOCK|O_CLOEXEC);
 819             if (vg_log_fd != -1) unlink(fname);
 820         }
 821     }
 822 #endif
 823
 824 #ifndef __WIN32__
 825     {
 826         bool colourise = true;
 827         const char *p = getenv("XAPIAN_TESTSUITE_OUTPUT");
 828         if (p == NULL || !*p || strcmp(p, "auto") == 0) {
 829             colourise = isatty(1);
 830         } else if (strcmp(p, "plain") == 0) {
 831             colourise = false;
 832         }
 833         if (colourise) {
 834             col_red = "\x1b[1m\x1b[31m";
 835             col_green = "\x1b[1m\x1b[32m";
 836             col_yellow = "\x1b[1m\x1b[33m";
 837             col_reset = "\x1b[0m";
 838             use_cr = true;
 839         }
 840     }
 841 #endif
 842
 843     static const struct option long_opts[] = {
 844         {"verbose",             no_argument, 0, 'v'},
 845         {"abort-on-error",      no_argument, 0, 'o'},
 846         {"help",                no_argument, 0, 'h'},
 847         {NULL,                  0, 0, 0}
 848     };
 849
 850     string short_opts_string = "voh";
 851     map<int, string *>::const_iterator i;
 852     for (i = short_opts.begin(); i != short_opts.end(); ++i) {
 853         short_opts_string += char(i->first);
 854         short_opts_string += ':';
 855     }
 856     const char * opts = short_opts_string.c_str();
 857
 858     int c;
 859     while ((c = gnu_getopt_long(argc, argv, opts, long_opts, 0)) != -1) {
 860         switch (c) {
 861             case 'v':
 862                 ++verbose;
 863                 break;
 864             case 'o':
 865                 abort_on_error = true;
 866                 break;
 867             default: {
 868                 i = short_opts.find(c);
 869                 if (i != short_opts.end()) {
 870                     i->second->assign(optarg);
 871                     break;
 872                 }
 873                 // -h or unrecognised option
 874                 usage();
 875                 return; // usage() doesn't return ...
 876             }
 877         }
 878     }
 879
 880     if (verbose == 0) {
 881         const char *p = getenv("VERBOSE");
 882         if (p != NULL) {
 883             verbose = atoi(p);
 884         }
 885     }
 886
 887     while (argv[optind]) {
 888         test_names.push_back(string(argv[optind]));
 889         optind++;
 890     }
 891 }
 892
 893 int
 894 test_driver::run(const test_desc *tests)
 895 {
 896     test_driver driver(tests);
 897
 898     test_driver::result myresult;
 899     myresult = driver.run_tests(test_names.begin(), test_names.end());
 900
 901     subtotal += myresult;
 902
 903     // Return value is a Unix-style exit code, so 0 for success and 1 for
 904     // failure.
 905     return myresult.failed > 0 || myresult.xpassed > 0;
 906 }
 907
 908 bool
 909 TEST_EQUAL_DOUBLE_(double a, double b)
 910 {
 911     if (a == b) return true;
 912     return (ceil(log10(max(fabs(a), fabs(b)))) - log10(fabs(a - b)) > DBL_DIG);
 913 }