xapian-letor/tests/harness/testsuite.cc

   1 /* testsuite.cc: a test suite engine
   2  *
   3  * Copyright 1999,2000,2001 BrightStation PLC
   4  * Copyright 2002 Ananova Ltd
   5  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2012,2013,2015,2016,2017 Olly Betts
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  20  * USA
  21  */
  22
  23 #include <config.h>
  24
  25 #include "testsuite.h"
  26
  27 #ifndef NO_LIBXAPIAN
  28 # include "backendmanager.h"
  29 #endif
  30 #include "fdtracker.h"
  31 #include "testrunner.h"
  32 #include "safeunistd.h"
  33
  34 #ifdef HAVE_VALGRIND
  35 # include <valgrind/memcheck.h>
  36 # include <sys/types.h>
  37 # include "safefcntl.h"
  38 #endif
  39
  40 #include <algorithm>
  41 #include <iostream>
  42 #include <set>
  43
  44 #include <cerrno>
  45 #include <cfloat> // For DBL_DIG.
  46 #include <cmath> // For ceil, fabs, log10.
  47 #include <cstdio>
  48 #include <cstdlib>
  49 #include <cstring>
  50
  51 #include "gnu_getopt.h"
  52
  53 #include <setjmp.h>
  54 #include <signal.h>
  55
  56 #include <exception>
  57 #ifdef USE_RTTI
  58 # include <typeinfo>
  59 # ifdef HAVE_CXXABI_H
  60 #  include <cxxabi.h>
  61 # endif
  62 #endif
  63
  64 #ifndef NO_LIBXAPIAN
  65 # include <xapian/error.h>
  66 #endif
  67 #include "filetests.h"
  68 #include "stringutils.h"
  69
  70 using namespace std;
  71
  72 /// The global verbose flag.
  73 int verbose;
  74
  75 #ifdef HAVE_VALGRIND
  76 static int vg_log_fd = -1;
  77 #endif
  78
  79 #ifdef HAVE_SIGSETJMP
  80 # define SIGSETJMP(ENV, SAVESIGS) sigsetjmp(ENV, SAVESIGS)
  81 # define SIGLONGJMP(ENV, VAL) siglongjmp(ENV, VAL)
  82 # define SIGJMP_BUF sigjmp_buf
  83 #else
  84 # define SIGSETJMP(ENV, SAVESIGS) setjmp(ENV)
  85 # define SIGLONGJMP(ENV, VAL) longjmp(ENV, VAL)
  86 # define SIGJMP_BUF jmp_buf
  87 #endif
  88
  89 /// The exception type we were expecting in TEST_EXCEPTION
  90 //  We use this to attempt to diagnose when the code fails to catch an
  91 //  exception when it should (due to a compiler or runtime fault in
  92 //  GCC 2.95 it seems)
  93 const char * expected_exception = NULL;
  94
  95 /// The debug printing stream
  96 std::ostringstream tout;
  97
  98 int test_driver::runs = 0;
  99 test_driver::result test_driver::subtotal;
 100 test_driver::result test_driver::total;
 101 string test_driver::argv0;
 102 string test_driver::opt_help;
 103 map<int, string *> test_driver::short_opts;
 104 vector<string> test_driver::test_names;
 105 bool test_driver::abort_on_error = false;
 106 string test_driver::col_red, test_driver::col_green;
 107 string test_driver::col_yellow, test_driver::col_reset;
 108 bool test_driver::use_cr = false;
 109
 110 void
 111 test_driver::write_and_clear_tout()
 112 {
 113     const string & s = tout.str();
 114     if (!s.empty()) {
 115         out << '\n' << s;
 116         tout.str(string());
 117     }
 118 }
 119
 120 string
 121 test_driver::get_srcdir()
 122 {
 123     char *p = getenv("srcdir");
 124     if (p != NULL) return string(p);
 125
 126 #ifdef __WIN32__
 127     // The path on argv[0] will always use \ for the directory separator.
 128     const char ARGV0_SEP = '\\';
 129 #else
 130     const char ARGV0_SEP = '/';
 131 #endif
 132     // Default srcdir to the pathname of argv[0].
 133     string srcdir(argv0);
 134     string::size_type i = srcdir.find_last_of(ARGV0_SEP);
 135     string srcfile;
 136     if (i != string::npos) {
 137         srcfile.assign(srcdir, i + 1, string::npos);
 138         srcdir.erase(i);
 139         // libtool may put the real executable in .libs.
 140         i = srcdir.find_last_of(ARGV0_SEP);
 141         if (srcdir.substr(i + 1) == ".libs") {
 142             srcdir.erase(i);
 143             // And it may have an "lt-" prefix.
 144             if (startswith(srcfile, "lt-")) srcfile.erase(0, 3);
 145         }
 146     } else {
 147         // No path of argv[0], so default srcdir to the current directory.
 148         // This may not work if libtool is involved as the true executable is
 149         // sometimes in ".libs".
 150         srcfile = srcdir;
 151         srcdir = ".";
 152     }
 153
 154     // Remove any trailing ".exe" suffix, since some platforms add this.
 155     if (endswith(srcfile, ".exe")) srcfile.resize(srcfile.size() - 4);
 156
 157     // Sanity check.
 158     if (!file_exists(srcdir + '/' + srcfile + ".cc")) {
 159         cout << argv0
 160              << ": srcdir is not in the environment and I can't guess it!\n"
 161                 "Run test programs using the runtest script - see HACKING for details"
 162              << endl;
 163         exit(1);
 164     }
 165     return srcdir;
 166 }
 167
 168 test_driver::test_driver(const test_desc *tests_)
 169         : out(cout.rdbuf()), tests(tests_)
 170 {
 171 }
 172
 173 static SIGJMP_BUF jb;
 174 static int signum = 0;
 175 static void * sigaddr = NULL;
 176
 177 // Needs C linkage so we can pass it to sigaction()/signal() without problems.
 178 extern "C" {
 179
 180 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 181 [[noreturn]]
 182 static void handle_sig(int signum_, siginfo_t *si, void *)
 183 {
 184     // Disable all our signal handlers to avoid problems if the signal
 185     // handling code causes a signal.
 186     struct sigaction sa;
 187     sa.sa_handler = SIG_DFL;
 188     sigemptyset(&sa.sa_mask);
 189     sa.sa_flags = 0;
 190     // We set the handlers with SA_RESETHAND, but that will only reset the
 191     // handler for the signal which fired.
 192     if (signum_ != SIGSEGV) sigaction(SIGSEGV, &sa, NULL);
 193     if (signum_ != SIGFPE) sigaction(SIGFPE, &sa, NULL);
 194     if (signum_ != SIGILL) sigaction(SIGILL, &sa, NULL);
 195 # ifdef SIGBUS
 196     if (signum_ != SIGBUS) sigaction(SIGBUS, &sa, NULL);
 197 # endif
 198 # ifdef SIGSTKFLT
 199     if (signum_ != SIGSTKFLT) sigaction(SIGSTKFLT, &sa, NULL);
 200 # endif
 201     signum = signum_;
 202     sigaddr = si->si_addr;
 203     SIGLONGJMP(jb, 1);
 204 }
 205
 206 #else
 207
 208 [[noreturn]]
 209 static void handle_sig(int signum_)
 210 {
 211     // Disable all our signal handlers to avoid problems if the signal
 212     // handling code causes a signal.
 213     signal(SIGSEGV, SIG_DFL);
 214     signal(SIGFPE, SIG_DFL);
 215     signal(SIGILL, SIG_DFL);
 216 #ifdef SIGBUS
 217     signal(SIGBUS, SIG_DFL);
 218 #endif
 219 #ifdef SIGSTKFLT
 220     signal(SIGSTKFLT, SIG_DFL);
 221 #endif
 222     signum = signum_;
 223     SIGLONGJMP(jb, 1);
 224 }
 225 #endif
 226
 227 }
 228
 229 class SignalRedirector {
 230   private:
 231     bool active;
 232   public:
 233     SignalRedirector() : active(false) { }
 234     void activate() {
 235         active = true;
 236         signum = 0;
 237         sigaddr = NULL;
 238         // SA_SIGINFO not universal (e.g. not present on Linux < 2.2 and Hurd).
 239 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 240         struct sigaction sa;
 241         sa.sa_sigaction = handle_sig;
 242         sigemptyset(&sa.sa_mask);
 243         sa.sa_flags = SA_RESETHAND|SA_SIGINFO;
 244         sigaction(SIGSEGV, &sa, NULL);
 245         sigaction(SIGFPE, &sa, NULL);
 246         sigaction(SIGILL, &sa, NULL);
 247 # ifdef SIGBUS
 248         sigaction(SIGBUS, &sa, NULL);
 249 # endif
 250 # ifdef SIGSTKFLT
 251         sigaction(SIGSTKFLT, &sa, NULL);
 252 # endif
 253 #else
 254         signal(SIGSEGV, handle_sig);
 255         signal(SIGFPE, handle_sig);
 256         signal(SIGILL, handle_sig);
 257 # ifdef SIGBUS
 258         signal(SIGBUS, handle_sig);
 259 # endif
 260 # ifdef SIGSTKFLT
 261         signal(SIGSTKFLT, handle_sig);
 262 # endif
 263 #endif
 264     }
 265     ~SignalRedirector() {
 266         if (active) {
 267 #if defined HAVE_SIGACTION && defined SA_SIGINFO
 268             struct sigaction sa;
 269             sa.sa_handler = SIG_DFL;
 270             sigemptyset(&sa.sa_mask);
 271             sa.sa_flags = 0;
 272             sigaction(SIGSEGV, &sa, NULL);
 273             sigaction(SIGFPE, &sa, NULL);
 274             sigaction(SIGILL, &sa, NULL);
 275 # ifdef SIGBUS
 276             sigaction(SIGBUS, &sa, NULL);
 277 # endif
 278 # ifdef SIGSTKFLT
 279             sigaction(SIGSTKFLT, &sa, NULL);
 280 # endif
 281 #else
 282             signal(SIGSEGV, SIG_DFL);
 283             signal(SIGFPE, SIG_DFL);
 284             signal(SIGILL, SIG_DFL);
 285 # ifdef SIGBUS
 286             signal(SIGBUS, SIG_DFL);
 287 # endif
 288 # ifdef SIGSTKFLT
 289             signal(SIGSTKFLT, SIG_DFL);
 290 # endif
 291 #endif
 292         }
 293     }
 294 };
 295
 296 //  A wrapper around the tests to trap exceptions,
 297 //  and avoid having to catch them in every test function.
 298 //  If this test driver is used for anything other than
 299 //  Xapian tests, then this ought to be provided by
 300 //  the client, really.
 301 //  return: test_driver::PASS, test_driver::FAIL, or test_driver::SKIP
 302 test_driver::test_result
 303 test_driver::runtest(const test_desc *test)
 304 {
 305     // This is used to make a note of how many times we've run the test
 306     volatile int runcount = 0;
 307
 308     FDTracker fdtracker;
 309     fdtracker.init();
 310
 311     while (true) {
 312         tout.str(string());
 313         if (SIGSETJMP(jb, 1) == 0) {
 314             SignalRedirector sig; // use object so signal handlers are reset
 315             static bool catch_signals =
 316                 (getenv("XAPIAN_TESTSUITE_SIG_DFL") == NULL);
 317             if (catch_signals) sig.activate();
 318             try {
 319                 expected_exception = NULL;
 320 #ifdef HAVE_VALGRIND
 321                 int vg_errs = 0;
 322                 long vg_leaks = 0, vg_dubious = 0, vg_reachable = 0;
 323                 if (vg_log_fd != -1) {
 324                     VALGRIND_DO_LEAK_CHECK;
 325                     vg_errs = VALGRIND_COUNT_ERRORS;
 326                     long dummy;
 327                     VALGRIND_COUNT_LEAKS(vg_leaks, vg_dubious, vg_reachable, dummy);
 328                     (void)dummy;
 329                     // Skip past any unread log output.
 330                     lseek(vg_log_fd, 0, SEEK_END);
 331                 }
 332 #endif
 333                 if (!test->run()) {
 334                     out << col_red << " FAILED" << col_reset;
 335                     write_and_clear_tout();
 336                     return FAIL;
 337                 }
 338                 if (verbose > 1)
 339                     write_and_clear_tout();
 340 #ifndef NO_LIBXAPIAN
 341                 if (backendmanager)
 342                     backendmanager->clean_up();
 343 #endif
 344 #ifdef HAVE_VALGRIND
 345                 if (vg_log_fd != -1) {
 346                     // We must empty tout before asking valgrind to perform its
 347                     // leak checks, otherwise the buffers holding the output
 348                     // may be identified as a memory leak (especially if >1K of
 349                     // output has been buffered it appears...)
 350                     tout.str(string());
 351 #define REPORT_FAIL_VG(M) do { \
 352     if (verbose) { \
 353         while (true) { \
 354             ssize_t c = read(vg_log_fd, buf, sizeof(buf)); \
 355             if (c == 0 || (c < 0 && errno != EINTR)) break; \
 356             if (c > 0) out << string(buf, c); \
 357         } \
 358     } \
 359     out << " " << col_red << M << col_reset; \
 360 } while (0)
 361                     // Record the current position so we can restore it so
 362                     // REPORT_FAIL_VG() gets the whole output.
 363                     off_t curpos = lseek(vg_log_fd, 0, SEEK_CUR);
 364                     char buf[4096];
 365                     while (true) {
 366                         ssize_t c = read(vg_log_fd, buf, sizeof(buf));
 367                         if (c == 0 || (c < 0 && errno != EINTR)) {
 368                             buf[0] = 0;
 369                             break;
 370                         }
 371                         if (c > 0) {
 372                             // Valgrind output has "==<pid>== \n" between
 373                             // report "records", so skip any lines like that,
 374                             // and also any warnings and continuation lines.
 375                             ssize_t i = 0;
 376                             while (true) {
 377                                 const char * spc;
 378                                 spc = static_cast<const char *>(
 379                                         memchr(buf + i, ' ', c - i));
 380                                 if (!spc) {
 381                                     i = c;
 382                                     break;
 383                                 }
 384                                 i = spc - buf;
 385                                 if (++i >= c) break;
 386                                 if (buf[i] == '\n')
 387                                     continue;
 388                                 if (c - i >= 8 &&
 389                                     (memcmp(buf + i, "Warning:", 8) == 0 ||
 390                                      memcmp(buf + i, "   ", 3) == 0)) {
 391                                     // Skip this line.
 392                                     i += 3;
 393                                     const char * nl;
 394                                     nl = static_cast<const char *>(
 395                                             memchr(buf + i, '\n', c - i));
 396                                     if (!nl) {
 397                                         i = c;
 398                                         break;
 399                                     }
 400                                     i = nl - buf;
 401                                     continue;
 402                                 }
 403                                 break;
 404                             }
 405
 406                             char *start = buf + i;
 407                             c -= i;
 408                             if (c > 128) c = 128;
 409
 410                             {
 411                                 const char *p;
 412                                 p = static_cast<const char*>(
 413                                         memchr(start, '\n', c));
 414                                 if (p != NULL) c = p - start;
 415                             }
 416
 417                             memmove(buf, start, c);
 418                             buf[c] = '\0';
 419                             break;
 420                         }
 421                     }
 422                     lseek(vg_log_fd, curpos, SEEK_SET);
 423
 424                     int vg_errs2 = VALGRIND_COUNT_ERRORS;
 425                     vg_errs = vg_errs2 - vg_errs;
 426                     VALGRIND_DO_LEAK_CHECK;
 427                     long vg_leaks2 = 0, vg_dubious2 = 0, vg_reachable2 = 0;
 428                     long dummy;
 429                     VALGRIND_COUNT_LEAKS(vg_leaks2, vg_dubious2, vg_reachable2,
 430                                          dummy);
 431                     (void)dummy;
 432                     vg_leaks = vg_leaks2 - vg_leaks;
 433                     vg_dubious = vg_dubious2 - vg_dubious;
 434                     vg_reachable = vg_reachable2 - vg_reachable;
 435                     if (vg_errs) {
 436                         string fail_msg(buf);
 437                         if (fail_msg.empty())
 438                             fail_msg = "VALGRIND DETECTED A PROBLEM";
 439                         REPORT_FAIL_VG(fail_msg);
 440                         return FAIL;
 441                     }
 442                     if (vg_leaks > 0) {
 443                         REPORT_FAIL_VG("LEAKED " << vg_leaks << " BYTES");
 444                         return FAIL;
 445                     }
 446                     if (vg_dubious > 0) {
 447                         // If code deliberately holds onto blocks by a pointer
 448                         // not to the start (e.g. languages/utilities.c does)
 449                         // then we need to rerun the test to see if the leak is
 450                         // real...
 451                         if (runcount == 0) {
 452                             out << col_yellow << " PROBABLY LEAKED MEMORY - RETRYING TEST" << col_reset;
 453                             ++runcount;
 454                             // Ensure that any cached memory from fd tracking
 455                             // is allocated before we rerun the test.
 456                             (void)fdtracker.check();
 457                             continue;
 458                         }
 459                         REPORT_FAIL_VG("PROBABLY LEAKED " << vg_dubious << " BYTES");
 460                         return FAIL;
 461                     }
 462                     if (vg_reachable > 0) {
 463                         // C++ STL implementations often "horde" released
 464                         // memory - for GCC 3.4 and newer the runtest script
 465                         // sets GLIBCXX_FORCE_NEW=1 which will disable this
 466                         // behaviour so we avoid this issue, but for older
 467                         // GCC and other compilers this may be an issue.
 468                         //
 469                         // See also:
 470                         // http://valgrind.org/docs/FAQ/#faq.reports
 471                         //
 472                         // For now, just use runcount to rerun the test and see
 473                         // if more is leaked - hopefully this shouldn't give
 474                         // false positives.
 475                         if (runcount == 0) {
 476                             out << col_yellow << " POSSIBLE UNRELEASED MEMORY - RETRYING TEST" << col_reset;
 477                             ++runcount;
 478                             // Ensure that any cached memory from fd tracking
 479                             // is allocated before we rerun the test.
 480                             (void)fdtracker.check();
 481                             continue;
 482                         }
 483                         REPORT_FAIL_VG("FAILED TO RELEASE " << vg_reachable << " BYTES");
 484                         return FAIL;
 485                     }
 486                 }
 487 #endif
 488                 if (!fdtracker.check()) {
 489                     if (runcount == 0) {
 490                         out << col_yellow << " POSSIBLE FDLEAK:" << fdtracker.get_message() << col_reset;
 491                         ++runcount;
 492                         continue;
 493                     }
 494                     out << col_red << " FDLEAK:" << fdtracker.get_message() << col_reset;
 495                     return FAIL;
 496                 }
 497             } catch (const TestFail &) {
 498                 out << col_red << " FAILED" << col_reset;
 499                 write_and_clear_tout();
 500                 return FAIL;
 501             } catch (const TestSkip &) {
 502                 out << col_yellow << " SKIPPED" << col_reset;
 503                 write_and_clear_tout();
 504                 return SKIP;
 505 #ifndef NO_LIBXAPIAN
 506             } catch (const Xapian::Error &err) {
 507                 string errclass = err.get_type();
 508                 if (expected_exception && expected_exception == errclass) {
 509                     out << col_yellow << " C++ FAILED TO CATCH " << errclass << col_reset;
 510                     return SKIP;
 511                 }
 512                 if (errclass == "NetworkError" &&
 513                     strcmp(err.get_error_string(), strerror(ECHILD)) == 0) {
 514                     // ECHILD suggests we've run out of processes, and that's
 515                     // much more likely to be a system issue than a Xapian bug.
 516                     //
 517                     // We also see apparently spurious ECHILD on Debian
 518                     // buildds sometimes: https://bugs.debian.org/681941
 519                     out << col_yellow << " ECHILD in network code" << col_reset;
 520                     return SKIP;
 521                 }
 522                 out << " " << col_red << err.get_description() << col_reset;
 523                 write_and_clear_tout();
 524                 return FAIL;
 525 #endif
 526             } catch (const string & msg) {
 527                 out << col_red << " EXCEPTION std::string " << msg << col_reset;
 528                 write_and_clear_tout();
 529                 return FAIL;
 530             } catch (const std::exception & e) {
 531                 out << " " << col_red;
 532 #ifndef USE_RTTI
 533                 out << "std::exception";
 534 #else
 535                 const char * name = typeid(e).name();
 536 # ifdef HAVE_CXXABI_H__
 537                 // __cxa_demangle() apparently requires GCC >= 3.1.
 538                 // Demangle the name which GCC returns for type_info::name().
 539                 int status;
 540                 char * realname = abi::__cxa_demangle(name, NULL, 0, &status);
 541                 if (realname) {
 542                     out << realname;
 543                     free(realname);
 544                 } else {
 545                     out << name;
 546                 }
 547 # else
 548                 out << name;
 549 # endif
 550 #endif
 551                 out << ": " << e.what() << col_reset;
 552                 write_and_clear_tout();
 553                 return FAIL;
 554             } catch (const char * msg) {
 555                 out << col_red;
 556                 if (msg) {
 557                     out << " EXCEPTION char * " << msg;
 558                 } else {
 559                     out << " EXCEPTION (char*)NULL";
 560                 }
 561                 out << col_reset;
 562                 write_and_clear_tout();
 563                 return FAIL;
 564             } catch (...) {
 565                 out << col_red << " UNKNOWN EXCEPTION" << col_reset;
 566                 write_and_clear_tout();
 567                 return FAIL;
 568             }
 569             return PASS;
 570         }
 571
 572         // Caught a signal.
 573         const char *signame = "SIGNAL";
 574         bool show_addr = true;
 575         switch (signum) {
 576             case SIGSEGV: signame = "SIGSEGV"; break;
 577             case SIGFPE: signame = "SIGFPE"; break;
 578             case SIGILL: signame = "SIGILL"; break;
 579 #ifdef SIGBUS
 580             case SIGBUS: signame = "SIGBUS"; break;
 581 #endif
 582 #ifdef SIGSTKFLT
 583             case SIGSTKFLT:
 584                 signame = "SIGSTKFLT";
 585                 show_addr = false;
 586                 break;
 587 #endif
 588         }
 589         out << " " << col_red << signame;
 590         if (show_addr) {
 591             char buf[40];
 592             sprintf(buf, " at %p", sigaddr);
 593             out << buf;
 594         }
 595         out << col_reset;
 596         write_and_clear_tout();
 597         return FAIL;
 598     }
 599 }
 600
 601 test_driver::result
 602 test_driver::run_tests(vector<string>::const_iterator b,
 603                        vector<string>::const_iterator e)
 604 {
 605     return do_run_tests(b, e);
 606 }
 607
 608 test_driver::result
 609 test_driver::run_tests()
 610 {
 611     const vector<string> blank;
 612     return do_run_tests(blank.begin(), blank.end());
 613 }
 614
 615 test_driver::result
 616 test_driver::do_run_tests(vector<string>::const_iterator b,
 617                           vector<string>::const_iterator e)
 618 {
 619     set<string> m(b, e);
 620     bool check_name = !m.empty();
 621
 622     test_driver::result res;
 623
 624     for (const test_desc *test = tests; test->name; ++test) {
 625         bool do_this_test = !check_name;
 626         if (!do_this_test && m.find(test->name) != m.end())
 627             do_this_test = true;
 628         if (!do_this_test) {
 629             // if this test is "foo123" see if "foo" was listed
 630             // this way "./testprog foo" can run foo1, foo2, etc.
 631             string t = test->name;
 632             string::size_type i;
 633             i = t.find_last_not_of("0123456789") + 1;
 634             if (i != string::npos) {
 635                 t.resize(i);
 636                 if (m.find(t) != m.end()) do_this_test = true;
 637             }
 638         }
 639         if (do_this_test) {
 640             out << "Running test: " << test->name << "...";
 641             out.flush();
 642             test_driver::test_result test_res = runtest(test);
 643 #ifndef NO_LIBXAPIAN
 644             if (backendmanager)
 645                 backendmanager->clean_up();
 646 #endif
 647             switch (test_res) {
 648                 case PASS:
 649                     ++res.succeeded;
 650                     if (verbose || !use_cr) {
 651                         out << col_green << " ok" << col_reset << endl;
 652                     } else {
 653                         out << "\r                                                                               \r";
 654                     }
 655                     break;
 656                 case FAIL:
 657                     ++res.failed;
 658                     out << endl;
 659                     if (abort_on_error) {
 660                         throw "Test failed - aborting further tests";
 661                     }
 662                     break;
 663                 case SKIP:
 664                     ++res.skipped;
 665                     out << endl;
 666                     // ignore the result of this test.
 667                     break;
 668             }
 669         }
 670     }
 671     return res;
 672 }
 673
 674 void
 675 test_driver::usage()
 676 {
 677     cout << "Usage: " << argv0 << " [-v|--verbose] [-o|--abort-on-error] " << opt_help
 678          << "[TESTNAME]..." << endl;
 679     cout << "       " << argv0 << " [-h|--help]" << endl;
 680     exit(1);
 681 }
 682
 683 /* Needs C linkage so we can pass it to atexit() without problems. */
 684 extern "C" {
 685 // Call upon program exit if there's more than one test run.
 686 static void
 687 report_totals(void)
 688 {
 689     test_driver::report(test_driver::total, "total");
 690 }
 691 }
 692
 693 void
 694 test_driver::report(const test_driver::result &r, const string &desc)
 695 {
 696     // Report totals at the end if we reported two or more subtotals.
 697     if (++runs == 2) atexit(report_totals);
 698
 699     if (r.succeeded != 0 || r.failed != 0) {
 700         cout << argv0 << " " << desc << ": ";
 701
 702         if (r.failed == 0)
 703             cout << "All ";
 704
 705         cout << col_green << r.succeeded << col_reset << " tests passed";
 706
 707         if (r.failed != 0)
 708             cout << ", " << col_red << r.failed << col_reset << " failed";
 709
 710         if (r.skipped) {
 711             cout << ", " << col_yellow << r.skipped << col_reset
 712                  << " skipped." << endl;
 713         } else {
 714             cout << "." << endl;
 715         }
 716     }
 717 }
 718
 719 void
 720 test_driver::add_command_line_option(const string &l, char s, string * arg)
 721 {
 722     short_opts.insert(make_pair(int(s), arg));
 723     opt_help += "[-";
 724     opt_help += s;
 725     opt_help += ' ';
 726     opt_help += l;
 727     opt_help += "] ";
 728 }
 729
 730 void
 731 test_driver::parse_command_line(int argc, char **argv)
 732 {
 733     argv0 = argv[0];
 734
 735 #ifndef __WIN32__
 736     {
 737         bool colourise = true;
 738         const char *p = getenv("XAPIAN_TESTSUITE_OUTPUT");
 739         if (p == NULL || !*p || strcmp(p, "auto") == 0) {
 740             colourise = isatty(1);
 741         } else if (strcmp(p, "plain") == 0) {
 742             colourise = false;
 743         }
 744         if (colourise) {
 745             col_red = "\x1b[1m\x1b[31m";
 746             col_green = "\x1b[1m\x1b[32m";
 747             col_yellow = "\x1b[1m\x1b[33m";
 748             col_reset = "\x1b[0m";
 749             use_cr = true;
 750         }
 751     }
 752 #endif
 753
 754     const struct option long_opts[] = {
 755         {"verbose",             no_argument, 0, 'v'},
 756         {"abort-on-error",      no_argument, 0, 'o'},
 757         {"help",                no_argument, 0, 'h'},
 758         {NULL,                  0, 0, 0}
 759     };
 760
 761     string short_opts_string = "voh";
 762     map<int, string *>::const_iterator i;
 763     for (i = short_opts.begin(); i != short_opts.end(); ++i) {
 764         short_opts_string += char(i->first);
 765         short_opts_string += ':';
 766     }
 767     const char * opts = short_opts_string.c_str();
 768
 769     int c;
 770     while ((c = gnu_getopt_long(argc, argv, opts, long_opts, 0)) != -1) {
 771         switch (c) {
 772             case 'v':
 773                 ++verbose;
 774                 break;
 775             case 'o':
 776                 abort_on_error = true;
 777                 break;
 778             default: {
 779                 i = short_opts.find(c);
 780                 if (i != short_opts.end()) {
 781                     i->second->assign(optarg);
 782                     break;
 783                 }
 784                 // -h or unrecognised option
 785                 usage();
 786                 return; // usage() doesn't return ...
 787             }
 788         }
 789     }
 790
 791     if (verbose == 0) {
 792         const char *p = getenv("VERBOSE");
 793         if (p != NULL) {
 794             verbose = atoi(p);
 795         }
 796     }
 797
 798     while (argv[optind]) {
 799         test_names.push_back(string(argv[optind]));
 800         ++optind;
 801     }
 802
 803 #ifdef HAVE_VALGRIND
 804     if (RUNNING_ON_VALGRIND) {
 805         if (getenv("XAPIAN_TESTSUITE_VALGRIND") != NULL) {
 806             // Open the valgrind log file, and unlink it.
 807             char fname[64];
 808             sprintf(fname, ".valgrind.log.%lu",
 809                     static_cast<unsigned long>(getpid()));
 810             vg_log_fd = open(fname, O_RDONLY|O_NONBLOCK|O_CLOEXEC);
 811             if (vg_log_fd != -1) unlink(fname);
 812         }
 813     }
 814 #endif
 815 }
 816
 817 int
 818 test_driver::run(const test_desc *tests)
 819 {
 820     test_driver driver(tests);
 821
 822     test_driver::result myresult;
 823     myresult = driver.run_tests(test_names.begin(), test_names.end());
 824
 825     subtotal += myresult;
 826
 827     return bool(myresult.failed); // if 0, then everything passed
 828 }
 829
 830 bool
 831 TEST_EQUAL_DOUBLE_(double a, double b)
 832 {
 833     if (a == b) return true;
 834     return (ceil(log10(max(fabs(a), fabs(b)))) - log10(fabs(a - b)) > DBL_DIG);
 835 }