xapian-core/tests/harness/testsuite.cc

   1 /** @file
   2  * @brief a test suite engine
   3  */
   4 /* Copyright 1999,2000,2001 BrightStation PLC
   5  * Copyright 2002 Ananova Ltd
   6  * Copyright 2002-2024 Olly Betts
   7  * Copyright 2007 Richard Boulton
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License as
  11  * published by the Free Software Foundation; either version 2 of the
  12  * License, or (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  22  * USA
  23  */
  24
  25 #include <config.h>
  26
  27 #include "testsuite.h"
  28
  29 #ifndef NO_LIBXAPIAN
  30 # include "backendmanager.h"
  31 #endif
  32 #include "fdtracker.h"
  33 #include "testrunner.h"
  34 #include "safeunistd.h"
  35
  36 #ifdef HAVE_VALGRIND
  37 # include <valgrind/memcheck.h>
  38 # include <sys/types.h>
  39 # include "safefcntl.h"
  40 #endif
  41
  42 #include <algorithm>
  43 #include <chrono>
  44 #include <ios>
  45 #include <iostream>
  46 #include <set>
  47
  48 #include <cerrno>
  49 #include <cfloat> // For DBL_DIG.
  50 #include <cmath> // For ceil, fabs, log10.
  51 #include <cstdio>
  52 #include <cstdlib>
  53 #include <cstring>
  54
  55 #include "gnu_getopt.h"
  56
  57 #include "parseint.h"
  58 #include <setjmp.h>
  59 #include <signal.h>
  60
  61 #include <exception>
  62 #ifdef USE_RTTI
  63 # include <typeinfo>
  64 # ifdef HAVE_CXXABI_H
  65 #  include <cxxabi.h>
  66 # endif
  67 #endif
  68
  69 #ifndef NO_LIBXAPIAN
  70 # include <xapian/error.h>
  71 #endif
  72 #include "errno_to_string.h"
  73 #include "filetests.h"
  74 #include "str.h"
  75 #include "stringutils.h"
  76
  77 using namespace std;
  78
  79 /// The global verbose flag.
  80 int verbose;
  81
  82 #ifdef HAVE_VALGRIND
  83 static int vg_log_fd = -1;
  84 #endif
  85
  86 #if HAVE_DECL_SIGSETJMP && HAVE_DECL_SIGLONGJMP
  87 # define SIGSETJMP(ENV, SAVESIGS) sigsetjmp(ENV, SAVESIGS)
  88 # define SIGLONGJMP(ENV, VAL) siglongjmp(ENV, VAL)
  89 # define SIGJMP_BUF sigjmp_buf
  90 #else
  91 # define SIGSETJMP(ENV, SAVESIGS) setjmp(ENV)
  92 # define SIGLONGJMP(ENV, VAL) longjmp(ENV, VAL)
  93 # define SIGJMP_BUF jmp_buf
  94 #endif
  95
  96 /// The exception type we were expecting in TEST_EXCEPTION
  97 //  We use this to attempt to diagnose when the code fails to catch an
  98 //  exception when it should (due to a compiler or runtime fault in
  99 //  GCC 2.95 it seems)
 100 const char * expected_exception = NULL;
 101
 102 const char* expected_failure;
 103
 104 /// The debug printing stream
 105 std::ostringstream tout;
 106
 107 int test_driver::runs = 0;
 108 test_driver::result test_driver::subtotal;
 109 test_driver::result test_driver::total;
 110 string test_driver::argv0;
 111 string test_driver::opt_help;
 112 map<int, string *> test_driver::short_opts;
 113 vector<string> test_driver::test_names;
 114 bool test_driver::abort_on_error = false;
 115 string test_driver::col_red, test_driver::col_green;
 116 string test_driver::col_yellow, test_driver::col_reset;
 117 bool test_driver::use_cr = false;
 118
 119 // time constant in seconds to mark tests as slow or not
 120 static const double SLOW_TEST_THRESHOLD = 10.00;
 121
 122 // vector to store the slow tests
 123 static vector<pair<const char*, double>> slow_tests;
 124
 125 void
 126 test_driver::write_and_clear_tout()
 127 {
 128     const string & s = tout.str();
 129     if (!s.empty()) {
 130         out << '\n' << s;
 131         tout.str(string());
 132     }
 133 }
 134
 135 string
 136 test_driver::get_srcdir()
 137 {
 138     char *p = getenv("srcdir");
 139     if (p != NULL) return string(p);
 140
 141     // Default srcdir to the pathname of argv[0].
 142     string srcdir(argv0);
 143     string::size_type i = srcdir.find_last_of(DIR_SEPS);
 144     string srcfile;
 145     if (i != string::npos) {
 146         srcfile.assign(srcdir, i + 1, string::npos);
 147         srcdir.erase(i);
 148         // libtool may put the real executable in .libs.
 149         i = srcdir.find_last_of(DIR_SEPS);
 150         if (srcdir.substr(i + 1) == ".libs") {
 151             srcdir.erase(i);
 152             // And it may have an "lt-" prefix.
 153             if (startswith(srcfile, "lt-")) srcfile.erase(0, 3);
 154         }
 155     } else {
 156         // No path of argv[0], so default srcdir to the current directory.
 157         // This may not work if libtool is involved as the true executable is
 158         // sometimes in ".libs".
 159         srcfile = srcdir;
 160         srcdir = ".";
 161     }
 162
 163     // Remove any trailing ".exe" suffix, since some platforms add this.
 164     if (endswith(srcfile, ".exe")) srcfile.resize(srcfile.size() - 4);
 165
 166     // Sanity check.
 167     if (!file_exists(srcdir + '/' + srcfile + ".cc")) {
 168         cout << argv0
 169              << ": srcdir is not in the environment and I can't guess it!\n"
 170                 "Run test programs using the runtest script - see HACKING "
 171                 "for details\n";
 172         exit(1);
 173     }
 174     return srcdir;
 175 }
 176
 177 test_driver::test_driver(const test_desc *tests_)
 178         : out(cout.rdbuf()), tests(tests_)
 179 {
 180     tout << boolalpha;
 181 }
 182
 183 static SIGJMP_BUF jb;
 184 static int signum = 0;
 185 static void * sigaddr = NULL;
 186
 187 // Needs C linkage so we can pass it to sigaction()/signal() without problems.
 188 extern "C" {
 189
 190 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 191 [[noreturn]]
 192 static void handle_sig(int signum_, siginfo_t *si, void *)
 193 {
 194     // Disable all our signal handlers to avoid problems if the signal
 195     // handling code causes a signal.
 196     struct sigaction sa;
 197     sa.sa_handler = SIG_DFL;
 198     sigemptyset(&sa.sa_mask);
 199     sa.sa_flags = 0;
 200     // We set the handlers with SA_RESETHAND, but that will only reset the
 201     // handler for the signal which fired.
 202     if (signum_ != SIGSEGV) sigaction(SIGSEGV, &sa, NULL);
 203     if (signum_ != SIGFPE) sigaction(SIGFPE, &sa, NULL);
 204     if (signum_ != SIGILL) sigaction(SIGILL, &sa, NULL);
 205 # ifdef SIGBUS
 206     if (signum_ != SIGBUS) sigaction(SIGBUS, &sa, NULL);
 207 # endif
 208 # ifdef SIGPIPE
 209     if (signum_ != SIGPIPE) sigaction(SIGPIPE, &sa, NULL);
 210 # endif
 211 # ifdef SIGSTKFLT
 212     if (signum_ != SIGSTKFLT) sigaction(SIGSTKFLT, &sa, NULL);
 213 # endif
 214     signum = signum_;
 215     sigaddr = si->si_addr;
 216     SIGLONGJMP(jb, 1);
 217 }
 218
 219 #else
 220
 221 [[noreturn]]
 222 static void handle_sig(int signum_)
 223 {
 224     // Disable all our signal handlers to avoid problems if the signal
 225     // handling code causes a signal.
 226     signal(SIGSEGV, SIG_DFL);
 227     signal(SIGFPE, SIG_DFL);
 228     signal(SIGILL, SIG_DFL);
 229 #ifdef SIGBUS
 230     signal(SIGBUS, SIG_DFL);
 231 #endif
 232 #ifdef SIGPIPE
 233     signal(SIGPIPE, SIG_DFL);
 234 #endif
 235 #ifdef SIGSTKFLT
 236     signal(SIGSTKFLT, SIG_DFL);
 237 #endif
 238     signum = signum_;
 239     SIGLONGJMP(jb, 1);
 240 }
 241 #endif
 242
 243 }
 244
 245 class SignalRedirector {
 246   private:
 247     bool active;
 248   public:
 249     SignalRedirector() : active(false) { }
 250     void activate() {
 251         active = true;
 252         signum = 0;
 253         sigaddr = NULL;
 254         // SA_SIGINFO is not universal (e.g. not present on Linux < 2.2 or
 255         // older Hurd).  If we have it, we use it to report the address
 256         // associated with the signal (for signals where that makes sense).
 257 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 258         struct sigaction sa;
 259         sa.sa_sigaction = handle_sig;
 260         sigemptyset(&sa.sa_mask);
 261         sa.sa_flags = SA_RESETHAND|SA_SIGINFO;
 262         sigaction(SIGSEGV, &sa, NULL);
 263         sigaction(SIGFPE, &sa, NULL);
 264         sigaction(SIGILL, &sa, NULL);
 265 # ifdef SIGBUS
 266         sigaction(SIGBUS, &sa, NULL);
 267 # endif
 268 # ifdef SIGPIPE
 269         sigaction(SIGPIPE, &sa, NULL);
 270 # endif
 271 # ifdef SIGSTKFLT
 272         sigaction(SIGSTKFLT, &sa, NULL);
 273 # endif
 274 #else
 275         signal(SIGSEGV, handle_sig);
 276         signal(SIGFPE, handle_sig);
 277         signal(SIGILL, handle_sig);
 278 # ifdef SIGBUS
 279         signal(SIGBUS, handle_sig);
 280 # endif
 281 # ifdef SIGPIPE
 282         signal(SIGPIPE, handle_sig);
 283 # endif
 284 # ifdef SIGSTKFLT
 285         signal(SIGSTKFLT, handle_sig);
 286 # endif
 287 #endif
 288     }
 289     ~SignalRedirector() {
 290         if (active) {
 291 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 292             struct sigaction sa;
 293             sa.sa_handler = SIG_DFL;
 294             sigemptyset(&sa.sa_mask);
 295             sa.sa_flags = 0;
 296             sigaction(SIGSEGV, &sa, NULL);
 297             sigaction(SIGFPE, &sa, NULL);
 298             sigaction(SIGILL, &sa, NULL);
 299 # ifdef SIGBUS
 300             sigaction(SIGBUS, &sa, NULL);
 301 # endif
 302 # ifdef SIGPIPE
 303             sigaction(SIGPIPE, &sa, NULL);
 304 # endif
 305 # ifdef SIGSTKFLT
 306             sigaction(SIGSTKFLT, &sa, NULL);
 307 # endif
 308 #else
 309             signal(SIGSEGV, SIG_DFL);
 310             signal(SIGFPE, SIG_DFL);
 311             signal(SIGILL, SIG_DFL);
 312 # ifdef SIGBUS
 313             signal(SIGBUS, SIG_DFL);
 314 # endif
 315 # ifdef SIGPIPE
 316             signal(SIGPIPE, SIG_DFL);
 317 # endif
 318 # ifdef SIGSTKFLT
 319             signal(SIGSTKFLT, SIG_DFL);
 320 # endif
 321 #endif
 322         }
 323     }
 324 };
 325
 326 //  A wrapper around the tests to trap exceptions,
 327 //  and avoid having to catch them in every test function.
 328 //  If this test driver is used for anything other than
 329 //  Xapian tests, then this ought to be provided by
 330 //  the client, really.
 331 //  return: test_driver::PASS, test_driver::FAIL, test_driver::SKIP,
 332 //  test_driver::XFAIL or test_driver:XPASS.
 333 test_driver::test_result
 334 test_driver::runtest(const test_desc *test)
 335 {
 336     // This is used to make a note of how many times we've run the test
 337     volatile int runcount = 0;
 338
 339     FDTracker fdtracker;
 340     fdtracker.init();
 341
 342     while (true) {
 343         tout.str(string());
 344 #ifdef _MSC_VER
 345 // Suppress warning about _setjmp and C++ object destruction.  It's not ideal
 346 // that some destructors may not get called, but this is in the test harness
 347 // rather than production code, and overall it's more helpful for the test
 348 // harness to be able to report clearly which testcase triggered a signal.
 349 # pragma warning(push)
 350 # pragma warning(disable:4611)
 351 #endif
 352         if (SIGSETJMP(jb, 1) == 0) {
 353 #ifdef _MSC_VER
 354 # pragma warning(pop)
 355 #endif
 356             SignalRedirector sig; // use object so signal handlers are reset
 357             static bool catch_signals =
 358                 (getenv("XAPIAN_TESTSUITE_SIG_DFL") == NULL);
 359             if (catch_signals) sig.activate();
 360             try {
 361                 expected_exception = NULL;
 362                 expected_failure = NULL;
 363 #ifdef HAVE_VALGRIND
 364                 int vg_errs = 0;
 365                 long vg_leaks = 0, vg_dubious = 0, vg_reachable = 0;
 366                 if (vg_log_fd != -1) {
 367                     VALGRIND_DO_LEAK_CHECK;
 368                     vg_errs = VALGRIND_COUNT_ERRORS;
 369                     long dummy;
 370                     VALGRIND_COUNT_LEAKS(vg_leaks, vg_dubious, vg_reachable, dummy);
 371                     (void)dummy;
 372                     // Skip past any unread log output.
 373                     lseek(vg_log_fd, 0, SEEK_END);
 374                 }
 375 #endif
 376                 test->run();
 377                 if (verbose > 1)
 378                     write_and_clear_tout();
 379 #ifndef NO_LIBXAPIAN
 380                 if (backendmanager)
 381                     backendmanager->clean_up();
 382 #endif
 383 #ifdef HAVE_VALGRIND
 384                 if (vg_log_fd != -1) {
 385                     // We must empty tout before asking valgrind to perform its
 386                     // leak checks, otherwise the buffers holding the output
 387                     // may be identified as a memory leak (especially if >1K of
 388                     // output has been buffered it appears...)
 389                     tout.str(string());
 390 #define REPORT_FAIL_VG(M) do { \
 391     if (verbose) { \
 392         while (true) { \
 393             ssize_t c = read(vg_log_fd, buf, sizeof(buf)); \
 394             if (c == 0 || (c < 0 && errno != EINTR)) break; \
 395             if (c > 0) out << string(buf, c); \
 396         } \
 397     } \
 398     out << " " << col_red << M << col_reset; \
 399 } while (0)
 400                     // Record the current position so we can restore it so
 401                     // REPORT_FAIL_VG() gets the whole output.
 402                     off_t curpos = lseek(vg_log_fd, 0, SEEK_CUR);
 403                     char buf[4096];
 404                     while (true) {
 405                         ssize_t c = read(vg_log_fd, buf, sizeof(buf));
 406                         if (c == 0 || (c < 0 && errno != EINTR)) {
 407                             buf[0] = 0;
 408                             break;
 409                         }
 410                         if (c > 0) {
 411                             // Valgrind output has "==<pid>== \n" between
 412                             // report "records", so skip any lines like that,
 413                             // and also any warnings and continuation lines.
 414                             ssize_t i = 0;
 415                             while (true) {
 416                                 const char * spc;
 417                                 spc = static_cast<const char *>(
 418                                         memchr(buf + i, ' ', c - i));
 419                                 if (!spc) {
 420                                     i = c;
 421                                     break;
 422                                 }
 423                                 i = spc - buf;
 424                                 if (++i >= c) break;
 425                                 if (buf[i] == '\n')
 426                                     continue;
 427                                 if (c - i >= 8 &&
 428                                     (memcmp(buf + i, "Warning:", 8) == 0 ||
 429                                      memcmp(buf + i, "   ", 3) == 0)) {
 430                                     // Skip this line.
 431                                     i += 3;
 432                                     const char * nl;
 433                                     nl = static_cast<const char *>(
 434                                             memchr(buf + i, '\n', c - i));
 435                                     if (!nl) {
 436                                         i = c;
 437                                         break;
 438                                     }
 439                                     i = nl - buf;
 440                                     continue;
 441                                 }
 442                                 break;
 443                             }
 444
 445                             char *start = buf + i;
 446                             c -= i;
 447                             if (c > 128) c = 128;
 448
 449                             {
 450                                 const char *p;
 451                                 p = static_cast<const char*>(
 452                                         memchr(start, '\n', c));
 453                                 if (p != NULL) c = p - start;
 454                             }
 455
 456                             memmove(buf, start, c);
 457                             buf[c] = '\0';
 458                             break;
 459                         }
 460                     }
 461                     lseek(vg_log_fd, curpos, SEEK_SET);
 462
 463                     int vg_errs2 = VALGRIND_COUNT_ERRORS;
 464                     vg_errs = vg_errs2 - vg_errs;
 465                     VALGRIND_DO_LEAK_CHECK;
 466                     long vg_leaks2 = 0, vg_dubious2 = 0, vg_reachable2 = 0;
 467                     long dummy;
 468                     VALGRIND_COUNT_LEAKS(vg_leaks2, vg_dubious2, vg_reachable2,
 469                                          dummy);
 470                     (void)dummy;
 471                     vg_leaks = vg_leaks2 - vg_leaks;
 472                     vg_dubious = vg_dubious2 - vg_dubious;
 473                     vg_reachable = vg_reachable2 - vg_reachable;
 474                     if (vg_errs) {
 475                         string fail_msg(buf);
 476                         if (fail_msg.empty())
 477                             fail_msg = "VALGRIND DETECTED A PROBLEM";
 478                         REPORT_FAIL_VG(fail_msg);
 479                         return FAIL;
 480                     }
 481                     if (vg_leaks > 0) {
 482                         REPORT_FAIL_VG("LEAKED " << vg_leaks << " BYTES");
 483                         return FAIL;
 484                     }
 485                     if (vg_dubious > 0) {
 486                         // If code deliberately holds onto blocks by a pointer
 487                         // not to the start (e.g. languages/utilities.c does)
 488                         // then we need to rerun the test to see if the leak is
 489                         // real...
 490                         if (runcount == 0) {
 491                             out << col_yellow << " PROBABLY LEAKED MEMORY - RETRYING TEST" << col_reset;
 492                             runcount = runcount + 1;
 493                             // Ensure that any cached memory from fd tracking
 494                             // is allocated before we rerun the test.
 495                             (void)fdtracker.check();
 496                             continue;
 497                         }
 498                         REPORT_FAIL_VG("PROBABLY LEAKED " << vg_dubious << " BYTES");
 499                         return FAIL;
 500                     }
 501                     if (vg_reachable > 0) {
 502                         // C++ STL implementations often "horde" released
 503                         // memory - the runtest script sets GLIBCXX_FORCE_NEW=1
 504                         // which under GCC will disable this behaviour and so
 505                         // we avoid this issue, but for other compilers this
 506                         // may be an issue.
 507                         //
 508                         // See also:
 509                         // https://valgrind.org/docs/manual/faq.html#faq.reports
 510                         //
 511                         // For now, just use runcount to rerun the test and see
 512                         // if more is leaked - hopefully this shouldn't give
 513                         // false positives.
 514                         if (runcount == 0) {
 515                             out << col_yellow << " POSSIBLE UNRELEASED MEMORY - RETRYING TEST" << col_reset;
 516                             runcount = runcount + 1;
 517                             // Ensure that any cached memory from fd tracking
 518                             // is allocated before we rerun the test.
 519                             (void)fdtracker.check();
 520                             continue;
 521                         }
 522                         REPORT_FAIL_VG("FAILED TO RELEASE " << vg_reachable << " BYTES");
 523                         return FAIL;
 524                     }
 525                 }
 526 #endif
 527                 if (!fdtracker.check()) {
 528                     if (runcount == 0) {
 529                         out << col_yellow << " POSSIBLE FDLEAK:" << fdtracker.get_message() << col_reset;
 530                         runcount = runcount + 1;
 531                         continue;
 532                     }
 533                     out << col_red << " FDLEAK:" << fdtracker.get_message() << col_reset;
 534                     return FAIL;
 535                 }
 536             } catch (const TestFail &) {
 537                 out << ' ';
 538                 if (expected_failure) {
 539                     out << col_yellow << "XFAIL (" << expected_failure << ")";
 540                 } else {
 541                     out << col_red << "FAILED";
 542                 }
 543                 out << col_reset;
 544                 write_and_clear_tout();
 545                 return expected_failure ? XFAIL : FAIL;
 546             } catch (const TestSkip &) {
 547                 out << col_yellow << " SKIPPED" << col_reset;
 548                 write_and_clear_tout();
 549                 return SKIP;
 550 #ifndef NO_LIBXAPIAN
 551             } catch (const Xapian::Error &err) {
 552                 out << ' ';
 553                 string errclass = err.get_type();
 554                 if (expected_exception && expected_exception == errclass) {
 555                     out << col_yellow << "C++ FAILED TO CATCH " << errclass << col_reset;
 556                     return SKIP;
 557                 }
 558                 if (errclass == "NetworkError" &&
 559                     err.get_error_string() != NULL &&
 560                     err.get_error_string() == errno_to_string(ECHILD)) {
 561                     // ECHILD suggests we've run out of processes, and that's
 562                     // much more likely to be a system issue than a Xapian bug.
 563                     //
 564                     // We also see apparently spurious ECHILD on Debian
 565                     // buildds sometimes: https://bugs.debian.org/681941
 566                     out << col_yellow << "ECHILD in network code" << col_reset;
 567                     return SKIP;
 568                 }
 569
 570                 if (expected_failure) {
 571                     out << col_yellow << "XFAIL (" << expected_failure
 572                         << "): ";
 573                 } else {
 574                     out << col_red << "FAIL: ";
 575                 }
 576                 out << err.get_description() << col_reset;
 577                 write_and_clear_tout();
 578                 return expected_failure ? XFAIL : FAIL;
 579 #endif
 580             } catch (const string & msg) {
 581                 out << ' ';
 582                 if (expected_failure) {
 583                     out << col_yellow << "XFAIL (" << expected_failure
 584                         << "): ";
 585                 } else {
 586                     out << col_red << "FAIL: ";
 587                 }
 588                 out << "EXCEPTION std::string " << msg << col_reset;
 589                 write_and_clear_tout();
 590                 return expected_failure ? XFAIL : FAIL;
 591             } catch (const std::exception & e) {
 592                 out << ' ';
 593                 if (expected_failure) {
 594                     out << col_yellow << "XFAIL (" << expected_failure
 595                         << "): ";
 596                 } else {
 597                     out << col_red << "FAIL: ";
 598                 }
 599 #ifndef USE_RTTI
 600                 out << "std::exception";
 601 #else
 602                 const char * name = typeid(e).name();
 603 # ifdef HAVE_CXXABI_H
 604                 // __cxa_demangle() apparently requires GCC >= 3.1.
 605                 // Demangle the name which GCC returns for type_info::name().
 606                 int status;
 607                 char * realname = abi::__cxa_demangle(name, NULL, 0, &status);
 608                 if (realname) {
 609                     out << realname;
 610                     free(realname);
 611                 } else {
 612                     out << name;
 613                 }
 614 # else
 615                 out << name;
 616 # endif
 617 #endif
 618                 out << ": " << e.what() << col_reset;
 619                 write_and_clear_tout();
 620                 return expected_failure ? XFAIL : FAIL;
 621             } catch (const char * msg) {
 622                 out << ' ';
 623                 if (expected_failure) {
 624                     out << col_yellow << "XFAIL (" << expected_failure
 625                         << "): ";
 626                 } else {
 627                     out << col_red << "FAIL: ";
 628                 }
 629                 if (msg) {
 630                     out << "EXCEPTION char* " << msg;
 631                 } else {
 632                     out << "EXCEPTION (char*)NULL";
 633                 }
 634                 out << col_reset;
 635                 write_and_clear_tout();
 636                 return expected_failure ? XFAIL : FAIL;
 637             } catch (...) {
 638                 out << ' ';
 639                 if (expected_failure) {
 640                     out << col_yellow << "XFAIL (" << expected_failure
 641                         << "): ";
 642                 } else {
 643                     out << col_red << "FAIL: ";
 644                 }
 645                 out << "UNKNOWN EXCEPTION" << col_reset;
 646                 write_and_clear_tout();
 647                 return expected_failure ? XFAIL : FAIL;
 648             }
 649
 650             if (expected_failure) {
 651                 // Testcase marked as expected to fail but actually passed.
 652                 out << ' ' << col_red << "XPASS (" << expected_failure << ")"
 653                     << col_reset;
 654                 write_and_clear_tout();
 655                 return XPASS;
 656             }
 657             return PASS;
 658         }
 659
 660         // Caught a signal.
 661         const char *signame = "SIGNAL";
 662 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 663         bool show_addr = true;
 664 #else
 665         bool show_addr = false;
 666 #endif
 667         switch (signum) {
 668             case SIGSEGV: signame = "SIGSEGV"; break;
 669             case SIGFPE: signame = "SIGFPE"; break;
 670             case SIGILL: signame = "SIGILL"; break;
 671 #ifdef SIGBUS
 672             case SIGBUS: signame = "SIGBUS"; break;
 673 #endif
 674 #ifdef SIGPIPE
 675             case SIGPIPE:
 676                 signame = "SIGPIPE";
 677                 show_addr = false;
 678                 break;
 679 #endif
 680 #ifdef SIGSTKFLT
 681             case SIGSTKFLT:
 682                 signame = "SIGSTKFLT";
 683                 show_addr = false;
 684                 break;
 685 #endif
 686         }
 687         out << " " << col_red << signame;
 688         if (show_addr) {
 689             out << " at " << sigaddr;
 690         }
 691         out << col_reset;
 692         write_and_clear_tout();
 693         return FAIL;
 694     }
 695 }
 696
 697 test_driver::result
 698 test_driver::run_tests(vector<string>::const_iterator b,
 699                        vector<string>::const_iterator e)
 700 {
 701     return do_run_tests(b, e);
 702 }
 703
 704 test_driver::result
 705 test_driver::run_tests()
 706 {
 707     const vector<string> blank;
 708     return do_run_tests(blank.begin(), blank.end());
 709 }
 710
 711 test_driver::result
 712 test_driver::do_run_tests(vector<string>::const_iterator b,
 713                           vector<string>::const_iterator e)
 714 {
 715     set<string> m(b, e);
 716     bool check_name = !m.empty();
 717
 718     test_driver::result res;
 719
 720     for (const test_desc *test = tests; test->name; ++test) {
 721         bool do_this_test = !check_name;
 722         if (!do_this_test && m.find(test->name) != m.end())
 723             do_this_test = true;
 724         if (!do_this_test) {
 725             // if this test is "foo123" see if "foo" was listed
 726             // this way "./testprog foo" can run foo1, foo2, etc.
 727             string t = test->name;
 728             string::size_type i;
 729             i = t.find_last_not_of("0123456789") + 1;
 730             if (i != string::npos) {
 731                 t.resize(i);
 732                 if (m.find(t) != m.end()) do_this_test = true;
 733             }
 734         }
 735         if (do_this_test) {
 736             out << "Running test: " << test->name << "...";
 737             out.flush();
 738             auto starttime = chrono::high_resolution_clock::now();
 739             test_driver::test_result test_res = runtest(test);
 740             auto endtime = chrono::high_resolution_clock::now();
 741             auto test_duration = chrono::duration_cast<chrono::duration<double>>
 742                                  (endtime - starttime);
 743 #ifndef NO_LIBXAPIAN
 744             if (backendmanager)
 745                 backendmanager->clean_up();
 746 #endif
 747             switch (test_res) {
 748                 case PASS:
 749                     ++res.succeeded;
 750
 751                     if (test_duration.count() >= SLOW_TEST_THRESHOLD) {
 752                         slow_tests.emplace_back(test->name,
 753                                                 test_duration.count());
 754                     }
 755
 756                     if (verbose || !use_cr) {
 757                         out << col_green << " ok" << col_reset << '\n';
 758                     } else {
 759                         out << "\r                                        "
 760                                "                                       \r";
 761                     }
 762                     break;
 763                 case XFAIL:
 764                     ++res.xfailed;
 765                     out << '\n';
 766                     break;
 767                 case FAIL:
 768                     ++res.failed;
 769                     out << '\n';
 770                     if (abort_on_error) {
 771                         throw "Test failed - aborting further tests";
 772                     }
 773                     break;
 774                 case XPASS:
 775                     ++res.xpassed;
 776                     out << '\n';
 777                     if (abort_on_error) {
 778                         throw "Test marked as XFAIL passed - aborting further tests";
 779                     }
 780                     break;
 781                 case SKIP:
 782                     ++res.skipped;
 783                     out << '\n';
 784                     // ignore the result of this test.
 785                     break;
 786             }
 787         }
 788     }
 789     return res;
 790 }
 791
 792 void
 793 test_driver::usage()
 794 {
 795     cout << "Usage: " << argv0 << " [-v|--verbose] [-o|--abort-on-error] "
 796          << opt_help << "[TESTNAME]...\n"
 797             "       " << argv0 << " [-h|--help]\n";
 798     exit(1);
 799 }
 800
 801 /* Needs C linkage so we can pass it to atexit() without problems. */
 802 extern "C" {
 803 // Call upon program exit if there's more than one test run.
 804 static void
 805 report_totals(void)
 806 {
 807     test_driver::report(test_driver::total, "total");
 808 }
 809 }
 810
 811 void
 812 test_driver::report(const test_driver::result &r, const string &desc)
 813 {
 814     // Report totals at the end if we reported two or more subtotals.
 815     if (++runs == 2) atexit(report_totals);
 816
 817     if (r.succeeded != 0 || r.failed != 0) {
 818         cout << argv0 << " " << desc << ": ";
 819
 820         if (r.failed == 0 && r.xpassed == 0)
 821             cout << "All ";
 822
 823         cout << col_green << r.succeeded << col_reset << " tests passed";
 824
 825         if (r.failed != 0)
 826             cout << ", " << col_red << r.failed << col_reset << " failed";
 827
 828         if (r.xpassed != 0)
 829             cout << ", " << col_red << r.xpassed << col_reset
 830                  << " expected failures passed";
 831
 832         if (r.xfailed != 0)
 833             cout << ", " << col_yellow << r.xfailed << col_reset
 834                  << " expected failures";
 835
 836         if (r.skipped) {
 837             cout << ", " << col_yellow << r.skipped << col_reset
 838                  << " skipped.\n";
 839         } else {
 840             cout << ".\n";
 841         }
 842
 843         if (!slow_tests.empty()) {
 844             const char* sep = "Slow tests: ";
 845             for (auto& test : slow_tests) {
 846                 cout << sep << test.first << " (" << test.second << " s)";
 847                 sep = ", ";
 848             }
 849             cout << ".\n";
 850             slow_tests.clear();
 851         }
 852     }
 853 }
 854
 855 void
 856 test_driver::add_command_line_option(const string &l, char s, string * arg)
 857 {
 858     short_opts.insert(make_pair(int(s), arg));
 859     opt_help += "[-";
 860     opt_help += s;
 861     opt_help += ' ';
 862     opt_help += l;
 863     opt_help += "] ";
 864 }
 865
 866 void
 867 test_driver::parse_command_line(int argc, char **argv)
 868 {
 869     argv0 = argv[0];
 870
 871 #ifdef HAVE_VALGRIND
 872     if (RUNNING_ON_VALGRIND) {
 873         if (getenv("XAPIAN_TESTSUITE_VALGRIND") != NULL) {
 874             // Open the valgrind log file, and unlink it.
 875             string fname = ".valgrind.log." + str(getpid());
 876             vg_log_fd = open(fname.c_str(), O_RDONLY|O_NONBLOCK|O_CLOEXEC);
 877             if (vg_log_fd != -1) unlink(fname.c_str());
 878         }
 879     }
 880 #endif
 881
 882 #ifndef __WIN32__
 883     {
 884         bool colourise = true;
 885         const char *p = getenv("XAPIAN_TESTSUITE_OUTPUT");
 886         if (p == NULL || !*p || strcmp(p, "auto") == 0) {
 887             colourise = isatty(1);
 888         } else if (strcmp(p, "plain") == 0) {
 889             colourise = false;
 890         }
 891         if (colourise) {
 892             col_red = "\x1b[1m\x1b[31m";
 893             col_green = "\x1b[1m\x1b[32m";
 894             col_yellow = "\x1b[1m\x1b[33m";
 895             col_reset = "\x1b[0m";
 896             use_cr = true;
 897         }
 898     }
 899 #endif
 900
 901     static const struct option long_opts[] = {
 902         {"verbose",             no_argument, 0, 'v'},
 903         {"abort-on-error",      no_argument, 0, 'o'},
 904         {"help",                no_argument, 0, 'h'},
 905         {NULL,                  0, 0, 0}
 906     };
 907
 908     string short_opts_string = "voh";
 909     map<int, string *>::const_iterator i;
 910     for (i = short_opts.begin(); i != short_opts.end(); ++i) {
 911         short_opts_string += char(i->first);
 912         short_opts_string += ':';
 913     }
 914     const char * opts = short_opts_string.c_str();
 915
 916     int c;
 917     while ((c = gnu_getopt_long(argc, argv, opts, long_opts, 0)) != -1) {
 918         switch (c) {
 919             case 'v':
 920                 ++verbose;
 921                 break;
 922             case 'o':
 923                 abort_on_error = true;
 924                 break;
 925             default: {
 926                 i = short_opts.find(c);
 927                 if (i != short_opts.end()) {
 928                     i->second->assign(optarg);
 929                     break;
 930                 }
 931                 // -h or unrecognised option
 932                 usage();
 933                 return; // usage() doesn't return ...
 934             }
 935         }
 936     }
 937
 938     if (verbose == 0) {
 939         const char *p = getenv("VERBOSE");
 940         if (p && *p) {
 941             unsigned int temp;
 942             if (!parse_unsigned(p, temp)) {
 943                 throw "VERBOSE must be a non-negative integer";
 944             }
 945             verbose = temp;
 946         }
 947     }
 948
 949     while (argv[optind]) {
 950         test_names.push_back(string(argv[optind]));
 951         ++optind;
 952     }
 953 }
 954
 955 int
 956 test_driver::run(const test_desc *tests)
 957 {
 958     test_driver driver(tests);
 959
 960     test_driver::result myresult;
 961     myresult = driver.run_tests(test_names.begin(), test_names.end());
 962
 963     subtotal += myresult;
 964
 965     // Return value is a Unix-style exit code, so 0 for success and 1 for
 966     // failure.
 967     return myresult.failed > 0 || myresult.xpassed > 0;
 968 }
 969
 970 bool
 971 TEST_EQUAL_DOUBLE_(double a, double b)
 972 {
 973     if (a == b) return true;
 974     return (ceil(log10(max(fabs(a), fabs(b)))) - log10(fabs(a - b)) > DBL_DIG);
 975 }