tools/heapcheck/chrome_tests.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 ''' Runs various chrome tests through heapcheck_test.py.
   7
   8 Most of this code is copied from ../valgrind/chrome_tests.py.
   9 TODO(glider): put common functions to a standalone module.
  10 '''
  11
  12 import glob
  13 import logging
  14 import optparse
  15 import os
  16 import stat
  17 import sys
  18
  19 import logging_utils
  20 import path_utils
  21
  22 import common
  23 import heapcheck_test
  24
  25 class TestNotFound(Exception): pass
  26
  27 def Dir2IsNewer(dir1, dir2):
  28   if dir2 is None or not os.path.isdir(dir2):
  29     return False
  30   if dir1 is None or not os.path.isdir(dir1):
  31     return True
  32   return os.stat(dir2)[stat.ST_MTIME] > os.stat(dir1)[stat.ST_MTIME]
  33
  34 def FindNewestDir(dirs):
  35   newest_dir = None
  36   for dir in dirs:
  37     if Dir2IsNewer(newest_dir, dir):
  38       newest_dir = dir
  39   return newest_dir
  40
  41 def File2IsNewer(file1, file2):
  42   if file2 is None or not os.path.isfile(file2):
  43     return False
  44   if file1 is None or not os.path.isfile(file1):
  45     return True
  46   return os.stat(file2)[stat.ST_MTIME] > os.stat(file1)[stat.ST_MTIME]
  47
  48 def FindDirContainingNewestFile(dirs, file):
  49   """Searches for the directory containing the newest copy of |file|.
  50
  51   Args:
  52     dirs: A list of paths to the directories to search among.
  53     file: A string containing the file name to search.
  54
  55   Returns:
  56     The string representing the the directory containing the newest copy of
  57     |file|.
  58
  59   Raises:
  60     IOError: |file| was not found.
  61   """
  62   newest_dir = None
  63   newest_file = None
  64   for dir in dirs:
  65     the_file = os.path.join(dir, file)
  66     if File2IsNewer(newest_file, the_file):
  67       newest_dir = dir
  68       newest_file = the_file
  69   if newest_dir is None:
  70     raise IOError("cannot find file %s anywhere, have you built it?" % file)
  71   return newest_dir
  72
  73 class ChromeTests(object):
  74   '''This class is derived from the chrome_tests.py file in ../purify/.
  75   '''
  76
  77   def __init__(self, options, args, test):
  78     # The known list of tests.
  79     # Recognise the original abbreviations as well as full executable names.
  80     self._test_list = {
  81       "ash": self.TestAsh,              "aura_shell_unittests": self.TestAsh,
  82       "aura": self.TestAura,            "aura_unittests": self.TestAura,
  83       "base": self.TestBase,            "base_unittests": self.TestBase,
  84       "browser": self.TestBrowser,      "browser_tests": self.TestBrowser,
  85       "crypto": self.TestCrypto,        "crypto_unittests": self.TestCrypto,
  86       "googleurl": self.TestGURL,       "googleurl_unittests": self.TestGURL,
  87       "content": self.TestContent,      "content_unittests": self.TestContent,
  88       "courgette": self.TestCourgette,
  89       "courgette_unittests": self.TestCourgette,
  90       "ipc": self.TestIpc,              "ipc_tests": self.TestIpc,
  91       "layout": self.TestLayout,        "layout_tests": self.TestLayout,
  92       "media": self.TestMedia,          "media_unittests": self.TestMedia,
  93       "net": self.TestNet,              "net_unittests": self.TestNet,
  94       "printing": self.TestPrinting,    "printing_unittests": self.TestPrinting,
  95       "remoting": self.TestRemoting,    "remoting_unittests": self.TestRemoting,
  96       "startup": self.TestStartup,      "startup_tests": self.TestStartup,
  97       "sync": self.TestSync,            "sync_unit_tests": self.TestSync,
  98       "test_shell": self.TestTestShell, "test_shell_tests": self.TestTestShell,
  99       "unit": self.TestUnit,            "unit_tests": self.TestUnit,
 100       "views": self.TestViews,          "views_unittests": self.TestViews,
 101       "sql": self.TestSql,              "sql_unittests": self.TestSql,
 102       "ui_unit": self.TestUIUnit,       "ui_unittests": self.TestUIUnit,
 103       "gfx": self.TestGfx,              "gfx_unittests": self.TestGfx,
 104     }
 105
 106     if test not in self._test_list:
 107       raise TestNotFound("Unknown test: %s" % test)
 108
 109     self._options = options
 110     self._args = args
 111     self._test = test
 112
 113     script_dir = path_utils.ScriptDir()
 114
 115     # Compute the top of the tree (the "source dir") from the script dir (where
 116     # this script lives).  We assume that the script dir is in tools/heapcheck/
 117     # relative to the top of the tree.
 118     self._source_dir = os.path.dirname(os.path.dirname(script_dir))
 119
 120     # Since this path is used for string matching, make sure it's always
 121     # an absolute Unix-style path.
 122     self._source_dir = os.path.abspath(self._source_dir).replace('\\', '/')
 123
 124     heapcheck_test_script = os.path.join(script_dir, "heapcheck_test.py")
 125     self._command_preamble = [heapcheck_test_script]
 126
 127   def _DefaultCommand(self, module, exe=None, heapcheck_test_args=None):
 128     '''Generates the default command array that most tests will use.
 129
 130     Args:
 131       module: The module name (corresponds to the dir in src/ where the test
 132               data resides).
 133       exe: The executable name.
 134       heapcheck_test_args: additional arguments to append to the command line.
 135     Returns:
 136       A string with the command to run the test.
 137     '''
 138     if not self._options.build_dir:
 139       dirs = [
 140         os.path.join(self._source_dir, "xcodebuild", "Debug"),
 141         os.path.join(self._source_dir, "out", "Debug"),
 142       ]
 143       if exe:
 144         self._options.build_dir = FindDirContainingNewestFile(dirs, exe)
 145       else:
 146         self._options.build_dir = FindNewestDir(dirs)
 147
 148     cmd = list(self._command_preamble)
 149
 150     if heapcheck_test_args != None:
 151       for arg in heapcheck_test_args:
 152         cmd.append(arg)
 153     if exe:
 154       cmd.append(os.path.join(self._options.build_dir, exe))
 155       # Heapcheck runs tests slowly, so slow tests hurt more; show elapased time
 156       # so we can find the slowpokes.
 157       cmd.append("--gtest_print_time")
 158     if self._options.gtest_repeat:
 159       cmd.append("--gtest_repeat=%s" % self._options.gtest_repeat)
 160     return cmd
 161
 162   def Suppressions(self):
 163     '''Builds the list of available suppressions files.'''
 164     ret = []
 165     directory = path_utils.ScriptDir()
 166     suppression_file = os.path.join(directory, "suppressions.txt")
 167     if os.path.exists(suppression_file):
 168       ret.append(suppression_file)
 169     suppression_file = os.path.join(directory, "suppressions_linux.txt")
 170     if os.path.exists(suppression_file):
 171       ret.append(suppression_file)
 172     return ret
 173
 174   def Run(self):
 175     '''Runs the test specified by command-line argument --test.'''
 176     logging.info("running test %s" % (self._test))
 177     return self._test_list[self._test]()
 178
 179   def _ReadGtestFilterFile(self, name, cmd):
 180     '''Reads files which contain lists of tests to filter out with
 181     --gtest_filter and appends the command-line option to |cmd|.
 182
 183     Args:
 184       name: the test executable name.
 185       cmd: the test running command line to be modified.
 186     '''
 187     filters = []
 188     directory = path_utils.ScriptDir()
 189     gtest_filter_files = [
 190         os.path.join(directory, name + ".gtest-heapcheck.txt"),
 191         # TODO(glider): Linux vs. CrOS?
 192     ]
 193     logging.info("Reading gtest exclude filter files:")
 194     for filename in gtest_filter_files:
 195       # strip the leading absolute path (may be very long on the bot)
 196       # and the following / or \.
 197       readable_filename = filename.replace(self._source_dir, "")[1:]
 198       if not os.path.exists(filename):
 199         logging.info("  \"%s\" - not found" % readable_filename)
 200         continue
 201       logging.info("  \"%s\" - OK" % readable_filename)
 202       f = open(filename, 'r')
 203       for line in f.readlines():
 204         if line.startswith("#") or line.startswith("//") or line.isspace():
 205           continue
 206         line = line.rstrip()
 207         filters.append(line)
 208     gtest_filter = self._options.gtest_filter
 209     if len(filters):
 210       if gtest_filter:
 211         gtest_filter += ":"
 212         if gtest_filter.find("-") < 0:
 213           gtest_filter += "-"
 214       else:
 215         gtest_filter = "-"
 216       gtest_filter += ":".join(filters)
 217     if gtest_filter:
 218       cmd.append("--gtest_filter=%s" % gtest_filter)
 219
 220   def SimpleTest(self, module, name, heapcheck_test_args=None, cmd_args=None):
 221     '''Builds the command line and runs the specified test.
 222
 223     Args:
 224       module: The module name (corresponds to the dir in src/ where the test
 225               data resides).
 226       name: The executable name.
 227       heapcheck_test_args: Additional command line args for heap checker.
 228       cmd_args: Additional command line args for the test.
 229     '''
 230     cmd = self._DefaultCommand(module, name, heapcheck_test_args)
 231     supp = self.Suppressions()
 232     self._ReadGtestFilterFile(name, cmd)
 233     if cmd_args:
 234       cmd.extend(["--"])
 235       cmd.extend(cmd_args)
 236
 237     # Sets LD_LIBRARY_PATH to the build folder so external libraries can be
 238     # loaded.
 239     if (os.getenv("LD_LIBRARY_PATH")):
 240       os.putenv("LD_LIBRARY_PATH", "%s:%s" % (os.getenv("LD_LIBRARY_PATH"),
 241                                               self._options.build_dir))
 242     else:
 243       os.putenv("LD_LIBRARY_PATH", self._options.build_dir)
 244     return heapcheck_test.RunTool(cmd, supp, module)
 245
 246   def TestAsh(self):
 247     return self.SimpleTest("ash", "aura_shell_unittests")
 248
 249   def TestAura(self):
 250     return self.SimpleTest("aura", "aura_unittests")
 251
 252   def TestBase(self):
 253     return self.SimpleTest("base", "base_unittests")
 254
 255   def TestBrowser(self):
 256     return self.SimpleTest("chrome", "browser_tests")
 257
 258   def TestCrypto(self):
 259     return self.SimpleTest("crypto", "crypto_unittests")
 260
 261   def TestGURL(self):
 262     return self.SimpleTest("chrome", "googleurl_unittests")
 263
 264   def TestContent(self):
 265     return self.SimpleTest("content", "content_unittests")
 266
 267   def TestCourgette(self):
 268     return self.SimpleTest("courgette", "courgette_unittests")
 269
 270   def TestMedia(self):
 271     return self.SimpleTest("chrome", "media_unittests")
 272
 273   def TestPrinting(self):
 274     return self.SimpleTest("chrome", "printing_unittests")
 275
 276   def TestRemoting(self):
 277     return self.SimpleTest("chrome", "remoting_unittests")
 278
 279   def TestSync(self):
 280     return self.SimpleTest("chrome", "sync_unit_tests")
 281
 282   def TestIpc(self):
 283     return self.SimpleTest("ipc", "ipc_tests")
 284
 285   def TestNet(self):
 286     return self.SimpleTest("net", "net_unittests")
 287
 288   def TestStartup(self):
 289     # We don't need the performance results, we're just looking for pointer
 290     # errors, so set number of iterations down to the minimum.
 291     os.putenv("STARTUP_TESTS_NUMCYCLES", "1")
 292     logging.info("export STARTUP_TESTS_NUMCYCLES=1");
 293     return self.SimpleTest("chrome", "startup_tests")
 294
 295   def TestTestShell(self):
 296     return self.SimpleTest("webkit", "test_shell_tests")
 297
 298   def TestUnit(self):
 299     return self.SimpleTest("chrome", "unit_tests")
 300
 301   def TestViews(self):
 302     return self.SimpleTest("views", "views_unittests")
 303
 304   def TestSql(self):
 305     return self.SimpleTest("chrome", "sql_unittests")
 306
 307   def TestUIUnit(self):
 308     return self.SimpleTest("chrome", "ui_unittests")
 309
 310   def TestGfx(self):
 311     return self.SimpleTest("chrome", "gfx_unittests")
 312
 313   def TestLayoutChunk(self, chunk_num, chunk_size):
 314     '''Runs tests [chunk_num*chunk_size .. (chunk_num+1)*chunk_size).
 315
 316     Wrap around to beginning of list at end. If chunk_size is zero, run all
 317     tests in the list once. If a text file is given as argument, it is used as
 318     the list of tests.
 319     '''
 320     # Build the ginormous commandline in 'cmd'.
 321     # It's going to be roughly
 322     #  python heapcheck_test.py ... python run_webkit_tests.py ...
 323     # but we'll use the --indirect flag to heapcheck_test.py
 324     # to avoid heapchecking python.
 325     # Start by building the heapcheck_test.py commandline.
 326     cmd = self._DefaultCommand("webkit")
 327
 328     # Now build script_cmd, the run_webkits_tests.py commandline
 329     # Store each chunk in its own directory so that we can find the data later
 330     chunk_dir = os.path.join("layout", "chunk_%05d" % chunk_num)
 331     test_shell = os.path.join(self._options.build_dir, "test_shell")
 332     out_dir = os.path.join(path_utils.ScriptDir(), "latest")
 333     out_dir = os.path.join(out_dir, chunk_dir)
 334     if os.path.exists(out_dir):
 335       old_files = glob.glob(os.path.join(out_dir, "*.txt"))
 336       for f in old_files:
 337         os.remove(f)
 338     else:
 339       os.makedirs(out_dir)
 340
 341     script = os.path.join(self._source_dir, "webkit", "tools", "layout_tests",
 342                           "run_webkit_tests.py")
 343     script_cmd = ["python", script, "--run-singly", "-v",
 344                   "--noshow-results", "--time-out-ms=200000",
 345                   "--nocheck-sys-deps"]
 346
 347     # Pass build mode to run_webkit_tests.py.  We aren't passed it directly,
 348     # so parse it out of build_dir.  run_webkit_tests.py can only handle
 349     # the two values "Release" and "Debug".
 350     # TODO(Hercules): unify how all our scripts pass around build mode
 351     # (--mode / --target / --build_dir / --debug)
 352     if self._options.build_dir.endswith("Debug"):
 353       script_cmd.append("--debug");
 354     if (chunk_size > 0):
 355       script_cmd.append("--run-chunk=%d:%d" % (chunk_num, chunk_size))
 356     if len(self._args):
 357       # if the arg is a txt file, then treat it as a list of tests
 358       if os.path.isfile(self._args[0]) and self._args[0][-4:] == ".txt":
 359         script_cmd.append("--test-list=%s" % self._args[0])
 360       else:
 361         script_cmd.extend(self._args)
 362     self._ReadGtestFilterFile("layout", script_cmd)
 363
 364     # Now run script_cmd with the wrapper in cmd
 365     cmd.extend(["--"])
 366     cmd.extend(script_cmd)
 367     supp = self.Suppressions()
 368     return heapcheck_test.RunTool(cmd, supp, "layout")
 369
 370   def TestLayout(self):
 371     '''Runs the layout tests.'''
 372     # A "chunk file" is maintained in the local directory so that each test
 373     # runs a slice of the layout tests of size chunk_size that increments with
 374     # each run.  Since tests can be added and removed from the layout tests at
 375     # any time, this is not going to give exact coverage, but it will allow us
 376     # to continuously run small slices of the layout tests under purify rather
 377     # than having to run all of them in one shot.
 378     chunk_size = self._options.num_tests
 379     if (chunk_size == 0):
 380       return self.TestLayoutChunk(0, 0)
 381     chunk_num = 0
 382     chunk_file = os.path.join("heapcheck_layout_chunk.txt")
 383
 384     logging.info("Reading state from " + chunk_file)
 385     try:
 386       f = open(chunk_file)
 387       if f:
 388         str = f.read()
 389         if len(str):
 390           chunk_num = int(str)
 391         # This should be enough so that we have a couple of complete runs
 392         # of test data stored in the archive (although note that when we loop
 393         # that we almost guaranteed won't be at the end of the test list)
 394         if chunk_num > 10000:
 395           chunk_num = 0
 396         f.close()
 397     except IOError, (errno, strerror):
 398       logging.error("error reading from file %s (%d, %s)" % (chunk_file,
 399                     errno, strerror))
 400     ret = self.TestLayoutChunk(chunk_num, chunk_size)
 401
 402     # Wait until after the test runs to completion to write out the new chunk
 403     # number.  This way, if the bot is killed, we'll start running again from
 404     # the current chunk rather than skipping it.
 405     logging.info("Saving state to " + chunk_file)
 406     try:
 407       f = open(chunk_file, "w")
 408       chunk_num += 1
 409       f.write("%d" % chunk_num)
 410       f.close()
 411     except IOError, (errno, strerror):
 412       logging.error("error writing to file %s (%d, %s)" % (chunk_file, errno,
 413                     strerror))
 414
 415     # Since we're running small chunks of the layout tests, it's important to
 416     # mark the ones that have errors in them.  These won't be visible in the
 417     # summary list for long, but will be useful for someone reviewing this bot.
 418     return ret
 419
 420
 421 def main():
 422   if not sys.platform.startswith('linux'):
 423     logging.error("Heap checking works only on Linux at the moment.")
 424     return 1
 425   parser = optparse.OptionParser("usage: %prog -b <dir> -t <test> "
 426                                  "[-t <test> ...]")
 427   parser.disable_interspersed_args()
 428   parser.add_option("-b", "--build_dir",
 429                     help="the location of the output of the compiler output")
 430   parser.add_option("-t", "--test", action="append",
 431                     help="which test to run")
 432   parser.add_option("", "--gtest_filter",
 433                     help="additional arguments to --gtest_filter")
 434   parser.add_option("", "--gtest_repeat",
 435                     help="argument for --gtest_repeat")
 436   parser.add_option("-v", "--verbose", action="store_true", default=False,
 437                     help="verbose output - enable debug log messages")
 438   # My machine can do about 120 layout tests/hour in release mode.
 439   # Let's do 30 minutes worth per run.
 440   # The CPU is mostly idle, so perhaps we can raise this when
 441   # we figure out how to run them more efficiently.
 442   parser.add_option("-n", "--num_tests", default=60, type="int",
 443                     help="for layout tests: # of subtests per run.  0 for all.")
 444
 445   options, args = parser.parse_args()
 446
 447   if options.verbose:
 448     logging_utils.config_root(logging.DEBUG)
 449   else:
 450     logging_utils.config_root()
 451
 452   if not options.test or not len(options.test):
 453     parser.error("--test not specified")
 454
 455   for t in options.test:
 456     tests = ChromeTests(options, args, t)
 457     ret = tests.Run()
 458     if ret:
 459       return ret
 460   return 0
 461
 462
 463 if __name__ == "__main__":
 464   sys.exit(main())