xapian-core/tests/perftest/perftest.cc

   1 /** @file perftest.cc
   2  * @brief performance tests for Xapian.
   3  */
   4 /* Copyright 2008 Lemur Consulting Ltd
   5  * Copyright 2008,2009,2010,2012,2013,2015 Olly Betts
   6  *
   7  * This program is free software; you can redistribute it and/or
   8  * modify it under the terms of the GNU General Public License as
   9  * published by the Free Software Foundation; either version 2 of the
  10  * License, or (at your option) any later version.
  11  *
  12  * This program is distributed in the hope that it will be useful,
  13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  * GNU General Public License for more details.
  16  *
  17  * You should have received a copy of the GNU General Public License
  18  * along with this program; if not, write to the Free Software
  19  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
  20  * USA
  21  */
  22
  23 #include <config.h>
  24 #include "perftest.h"
  25
  26 #include "backendmanager.h"
  27 #include "freemem.h"
  28 #include "omassert.h"
  29 #include "perftest/perftest_all.h"
  30 #include "realtime.h"
  31 #include "runprocess.h"
  32 #include "str.h"
  33 #include "stringutils.h"
  34 #include "testrunner.h"
  35 #include "testsuite.h"
  36
  37 #include <cstdlib>
  38 #include <iostream>
  39
  40 #include "safeunistd.h"
  41 #ifdef HAVE_SYS_UTSNAME_H
  42 # include <sys/utsname.h>
  43 #endif
  44
  45 #ifdef __WIN32__
  46 # include "safewindows.h"
  47 # include "safewinsock2.h"
  48 #endif
  49
  50 using namespace std;
  51
  52 PerfTestLogger logger;
  53
  54 static string
  55 escape_xml(const string & str)
  56 {
  57     string res;
  58     string::size_type p = 0;
  59     while (p < str.size()) {
  60         char ch = str[p++];
  61         switch (ch) {
  62             case '<':
  63                 res += "&lt;";
  64                 continue;
  65             case '>':
  66                 res += "&gt;";
  67                 continue;
  68             case '&':
  69                 res += "&amp;";
  70                 continue;
  71             case '"':
  72                 res += "&quot;";
  73                 continue;
  74             default:
  75                 res += ch;
  76         }
  77     }
  78     return res;
  79 }
  80
  81 PerfTestLogger::PerfTestLogger()
  82         : testcase_started(false),
  83           indexing_started(false),
  84           searching_started(false)
  85 {}
  86
  87 PerfTestLogger::~PerfTestLogger()
  88 {
  89     close();
  90 }
  91
  92 /// Get the hostname.
  93 static string
  94 get_hostname()
  95 {
  96 #ifdef __WIN32__
  97     char buf[256];
  98     WORD WSAVerReq = MAKEWORD(1, 1);
  99     WSADATA WSAData;
 100
 101     if (WSAStartup(WSAVerReq, &WSAData) != 0) {
 102         // wrong winsock dlls?
 103         return string();
 104     }
 105     if (gethostname(buf, sizeof(buf)) != 0) {
 106         *buf = '\0';
 107     }
 108     WSACleanup();
 109     return buf;
 110 #elif defined HAVE_SYS_UTSNAME_H
 111     struct utsname uname_buf;
 112     if (uname(&uname_buf) != 0) {
 113         uname_buf.nodename[0] = '\0';
 114     }
 115     return uname_buf.nodename;
 116 #elif defined HAVE_GETHOSTNAME
 117     char buf[256];
 118     if (gethostname(buf, sizeof(buf)) != 0) {
 119         *buf = '\0';
 120     }
 121     return buf;
 122 #else
 123     return string();
 124 #endif
 125 }
 126
 127 /// Get the load average.
 128 static string
 129 get_loadavg()
 130 {
 131 #ifdef __WIN32__
 132     return string();
 133 #else
 134     string loadavg;
 135     try {
 136         loadavg = stdout_to_string("uptime 2>/dev/null | sed 's/.*: \\([0-9][0-9]*\\)/\\1/;s/, .*//'");
 137     } catch (NoSuchProgram) {} catch (ReadError) {}
 138     return loadavg;
 139 #endif
 140 }
 141
 142 /// Get the number of processors.
 143 static string
 144 get_ncpus()
 145 {
 146     string ncpus;
 147 #ifdef __WIN32__
 148     SYSTEM_INFO siSysInfo;
 149     GetSystemInfo(&siSysInfo);
 150     ncpus = str(siSysInfo.dwNumberOfProcessors);
 151 #else
 152     try {
 153         // Works on Linux, at least back to kernel 2.2.26.
 154         ncpus = stdout_to_string("getconf _NPROCESSORS_ONLN 2>/dev/null | grep -v '[^0-9]'");
 155     } catch (NoSuchProgram) {} catch (ReadError) {}
 156     if (ncpus.empty())
 157         try {
 158             // Works on OpenBSD (and apparently FreeBSD and Darwin).
 159             ncpus = stdout_to_string("sysctl hw.ncpu 2>/dev/null | sed 's/.*=//'");
 160         } catch (NoSuchProgram) {} catch (ReadError) {}
 161     if (ncpus.empty())
 162         try {
 163             // Works on Solaris and OSF/1.
 164             ncpus = stdout_to_string("PATH=/usr/sbin:$PATH psrinfo 2>/dev/null | grep -c on-line");
 165         } catch (NoSuchProgram) {} catch (ReadError) {}
 166     if (ncpus.empty())
 167         try {
 168             // Works on Linux, just in case the getconf version doesn't.
 169             // Different architectures have different formats for /proc/cpuinfo
 170             // so this won't work as widely as getconf _NPROCESSORS_ONLN will.
 171             ncpus = stdout_to_string("grep -c processor /proc/cpuinfo 2>/dev/null");
 172         } catch (NoSuchProgram) {} catch (ReadError) {}
 173 #endif
 174     return ncpus;
 175 }
 176
 177 /// Get details of the OS and distribution.
 178 static string
 179 get_distro()
 180 {
 181     string distro;
 182 #ifdef __WIN32__
 183     OSVERSIONINFO osvi;
 184     ZeroMemory(&osvi, sizeof(OSVERSIONINFO));
 185     osvi.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
 186     GetVersionEx(&osvi);
 187     distro = "Microsoft Windows v";
 188     distro += str(osvi.dwMajorVersion);
 189     distro += '.';
 190     distro += str(osvi.dwMinorVersion);
 191     distro += '.';
 192     distro += str(osvi.dwBuildNumber);
 193 #else
 194     try {
 195         distro = stdout_to_string("perftest/get_machine_info 2>/dev/null");
 196     } catch (NoSuchProgram) {} catch (ReadError) {}
 197 #endif
 198     return distro;
 199 }
 200
 201 /// Get the git commit for HEAD.
 202 static string
 203 get_commit_ref()
 204 {
 205     string commit_ref;
 206     try {
 207         commit_ref = stdout_to_string("cd \"$srcdir\" && git log -n1 --abbrev-commit --format=%h");
 208     } catch (NoSuchProgram) {} catch (ReadError) {}
 209
 210     return commit_ref;
 211 }
 212
 213 bool
 214 PerfTestLogger::open(const string & logpath)
 215 {
 216     out.open(logpath.c_str(), ios::out | ios::binary | ios::trunc);
 217     if (!out.is_open()) {
 218         cerr << "Couldn't open output logfile '" << logpath << "'" << endl;
 219         return false;
 220     }
 221
 222     string loadavg = get_loadavg();
 223     string hostname = get_hostname();
 224     string ncpus = get_ncpus();
 225     string distro = get_distro();
 226
 227     // Write header, and details of the machine.
 228     write("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<testrun>\n"
 229           " <machineinfo>\n");
 230     if (!hostname.empty())
 231         write("  <hostname>" + hostname + "</hostname>\n");
 232     if (!loadavg.empty())
 233         write("  <loadavg>" + loadavg + "</loadavg>\n");
 234     if (!ncpus.empty())
 235         write("  <ncpus>" + ncpus + "</ncpus>\n");
 236     if (!distro.empty())
 237         write("  <distro>" + distro + "</distro>\n");
 238     write("  <physmem>" + str(get_total_physical_memory()) + "</physmem>\n");
 239     write(" </machineinfo>\n");
 240
 241     const string & commit_ref = get_commit_ref();
 242
 243     write(" <sourceinfo>\n");
 244     if (!commit_ref.empty())
 245         write("  <commitref>" + commit_ref + "</commitref>\n");
 246     write("  <version>" + string(Xapian::version_string()) + "</version>\n");
 247     write(" </sourceinfo>\n");
 248
 249
 250     return true;
 251 }
 252
 253 void
 254 PerfTestLogger::write(const string & text)
 255 {
 256     out.write(text.data(), text.size());
 257     out.flush();
 258 }
 259
 260 void
 261 PerfTestLogger::close()
 262 {
 263     repetition_end();
 264     if (out.is_open()) {
 265         write("</testrun>\n");
 266         out.close();
 267     }
 268 }
 269
 270 void
 271 PerfTestLogger::indexing_begin(const string & dbname,
 272                                const std::map<std::string, std::string> & params)
 273 {
 274     searching_end();
 275     indexing_end();
 276     write("  <indexrun dbname=\"" + dbname + "\">\n   <params>\n");
 277     std::map<std::string, std::string>::const_iterator i;
 278     for (i = params.begin(); i != params.end(); ++i) {
 279         write("    <param name=\"" + i->first + "\">" + escape_xml(i->second) + "</param>\n");
 280     }
 281     i = params.find("flush_threshold");
 282     if (i == params.end()) {
 283         Xapian::doccount flush_threshold = 0;
 284         const char *p = getenv("XAPIAN_FLUSH_THRESHOLD");
 285         if (p)
 286             flush_threshold = atoi(p);
 287         if (flush_threshold == 0)
 288             flush_threshold = 10000;
 289         write("    <param name=\"flush_threshold\">" +
 290               escape_xml(str(flush_threshold)) + "</param>\n");
 291     }
 292     write("   </params>\n");
 293     indexing_addcount = 0;
 294     indexing_unlogged_changes = false;
 295     indexing_timer = RealTime::now();
 296     last_indexlog_timer = indexing_timer;
 297     indexing_started = true;
 298
 299     indexing_log();
 300 }
 301
 302 void
 303 PerfTestLogger::indexing_log()
 304 {
 305     Assert(indexing_started);
 306     last_indexlog_timer = RealTime::now();
 307     double elapsed(last_indexlog_timer - indexing_timer);
 308     write("   <item>"
 309           "<time>" + str(elapsed) + "</time>"
 310           "<adds>" + str(indexing_addcount) + "</adds>"
 311           "</item>\n");
 312     indexing_unlogged_changes = false;
 313 }
 314
 315 void
 316 PerfTestLogger::indexing_add()
 317 {
 318     ++indexing_addcount;
 319     indexing_unlogged_changes = true;
 320     // Log every 1000 documents
 321     if (indexing_addcount % 1000 == 0) {
 322         indexing_log();
 323     } else {
 324         // Or after 5 seconds
 325         double now = RealTime::now();
 326         if (now > last_indexlog_timer + 5)
 327             indexing_log();
 328     }
 329 }
 330
 331 void
 332 PerfTestLogger::indexing_end()
 333 {
 334     if (indexing_started) {
 335         indexing_log();
 336         write("  </indexrun>\n");
 337         indexing_started = false;
 338     }
 339 }
 340
 341 void
 342 PerfTestLogger::searching_start(const string & description)
 343 {
 344     indexing_end();
 345     searching_end();
 346     write("   <searchrun>\n"
 347           "    <description>" + escape_xml(description) + "</description>\n");
 348     searching_started = true;
 349     search_start();
 350 }
 351
 352 void
 353 PerfTestLogger::search_start()
 354 {
 355     searching_timer = RealTime::now();
 356 }
 357
 358 void
 359 PerfTestLogger::search_end(const Xapian::Query & query,
 360                            const Xapian::MSet & mset)
 361 {
 362     Assert(searching_started);
 363     double elapsed(RealTime::now() - searching_timer);
 364     write("    <search>"
 365           "<time>" + str(elapsed) + "</time>"
 366           "<query>" + escape_xml(query.get_description()) + "</query>"
 367           "<mset>"
 368           "<size>" + str(mset.size()) + "</size>"
 369           "<lb>" + str(mset.get_matches_lower_bound()) + "</lb>"
 370           "<est>" + str(mset.get_matches_estimated()) + "</est>"
 371           "<ub>" + str(mset.get_matches_upper_bound()) + "</ub>"
 372           "</mset>"
 373           "</search>\n");
 374     search_start();
 375 }
 376
 377 void
 378 PerfTestLogger::searching_end()
 379 {
 380     if (searching_started) {
 381         write("   </searchrun>\n");
 382         searching_started = false;
 383     }
 384 }
 385
 386 void
 387 PerfTestLogger::testcase_begin(const string & testcase)
 388 {
 389     testcase_end();
 390     write(" <testcase name=\"" + testcase + "\" backend=\"" +
 391           backendmanager->get_dbtype() + "\" repnum=\"" +
 392           str(repetition_number) + "\">\n");
 393     testcase_started = true;
 394 }
 395
 396 void
 397 PerfTestLogger::testcase_end()
 398 {
 399     indexing_end();
 400     if (testcase_started) {
 401         write(" </testcase>\n");
 402         testcase_started = false;
 403     }
 404 }
 405
 406 void
 407 PerfTestLogger::repetition_begin(int num)
 408 {
 409     repetition_end();
 410     repetition_number = num;
 411 }
 412
 413 void
 414 PerfTestLogger::repetition_end()
 415 {
 416     testcase_end();
 417 }
 418
 419
 420 class PerfTestRunner : public TestRunner
 421 {
 422     string repetitions_string;
 423     mutable bool repetitions_parsed;
 424     mutable int repetitions;
 425   public:
 426     PerfTestRunner()
 427             : repetitions_parsed(false), repetitions(5)
 428     {
 429         test_driver::add_command_line_option("repetitions", 'r',
 430                                              &repetitions_string);
 431     }
 432
 433     int run() const {
 434         int result = 0;
 435         if (!repetitions_parsed) {
 436             if (!repetitions_string.empty()) {
 437                 repetitions = atoi(repetitions_string.c_str());
 438             }
 439             repetitions_parsed = true;
 440         }
 441         for (int i = 0; i != repetitions; ++i) {
 442             logger.repetition_begin(i + 1);
 443 #include "perftest/perftest_collated.h"
 444             logger.repetition_end();
 445         }
 446         return result;
 447     }
 448 };
 449
 450 int main(int argc, char **argv)
 451 {
 452     if (!logger.open("perflog.xml"))
 453         return 1;
 454
 455     PerfTestRunner runner;
 456
 457     return runner.run_tests(argc, argv);
 458 }