tools/valgrind/scan-build.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 import argparse
   7 import errno
   8 import os
   9 import re
  10 import sys
  11 import urllib
  12 import urllib2
  13
  14 # Where all the data lives.
  15 ROOT_URL = "http://build.chromium.org/p/chromium.memory.fyi/builders"
  16
  17 # TODO(groby) - support multi-line search from the command line. Useful when
  18 # scanning for classes of failures, see below.
  19 SEARCH_STRING = """<p class=\"failure result\">
  20 Failed memory test: content
  21 </p>"""
  22
  23 # Location of the log cache.
  24 CACHE_DIR = "buildlogs.tmp"
  25
  26 # If we don't find anything after searching |CUTOFF| logs, we're probably done.
  27 CUTOFF = 100
  28
  29 def EnsurePath(path):
  30   """Makes sure |path| does exist, tries to create it if it doesn't."""
  31   try:
  32     os.makedirs(path)
  33   except OSError as exception:
  34     if exception.errno != errno.EEXIST:
  35       raise
  36
  37
  38 class Cache(object):
  39   def __init__(self, root_dir):
  40     self._root_dir = os.path.abspath(root_dir)
  41
  42   def _LocalName(self, name):
  43     """If name is a relative path, treat it as relative to cache root.
  44        If it is absolute and under cache root, pass it through.
  45        Otherwise, raise error.
  46     """
  47     if os.path.isabs(name):
  48       assert os.path.commonprefix([name, self._root_dir]) == self._root_dir
  49     else:
  50       name = os.path.join(self._root_dir, name)
  51     return name
  52
  53   def _FetchLocal(self, local_name):
  54     local_name = self._LocalName(local_name)
  55     EnsurePath(os.path.dirname(local_name))
  56     if os.path.exists(local_name):
  57       f = open(local_name, 'r')
  58       return f.readlines();
  59     return None
  60
  61   def _FetchRemote(self, remote_name):
  62     try:
  63       response = urllib2.urlopen(remote_name)
  64     except:
  65       print "Could not fetch", remote_name
  66       raise
  67     return response.read()
  68
  69   def Update(self, local_name, remote_name):
  70     local_name = self._LocalName(local_name)
  71     EnsurePath(os.path.dirname(local_name))
  72     blob = self._FetchRemote(remote_name)
  73     f = open(local_name, "w")
  74     f.write(blob)
  75     return blob.splitlines()
  76
  77   def FetchData(self, local_name, remote_name):
  78     result = self._FetchLocal(local_name)
  79     if result:
  80       return result
  81     # If we get here, the local cache does not exist yet. Fetch, and store.
  82     return self.Update(local_name, remote_name)
  83
  84
  85 class Builder(object):
  86   def __init__(self, waterfall, name):
  87     self._name = name
  88     self._waterfall = waterfall
  89
  90   def Name(self):
  91     return self._name
  92
  93   def LatestBuild(self):
  94     return self._waterfall.GetLatestBuild(self._name)
  95
  96   def GetBuildPath(self, build_num):
  97     return "%s/%s/builds/%d" % (
  98         self._waterfall._root_url, urllib.quote(self._name), build_num)
  99
 100   def _FetchBuildLog(self, build_num):
 101     local_build_path = "builds/%s" % self._name
 102     local_build_file = os.path.join(local_build_path, "%d.log" % build_num)
 103     return self._waterfall._cache.FetchData(local_build_file,
 104                                             self.GetBuildPath(build_num))
 105
 106   def _CheckLog(self, build_num, tester):
 107     log_lines = self._FetchBuildLog(build_num)
 108     return any(tester(line) for line in log_lines)
 109
 110   def ScanLogs(self, tester):
 111     occurrences = []
 112     build = self.LatestBuild()
 113     no_results = 0
 114     while build != 0 and no_results < CUTOFF:
 115       if self._CheckLog(build, tester):
 116         occurrences.append(build)
 117       else:
 118         no_results = no_results + 1
 119       build = build - 1
 120     return occurrences
 121
 122
 123 class Waterfall(object):
 124   def __init__(self, root_url, cache_dir):
 125     self._root_url = root_url
 126     self._builders = {}
 127     self._top_revision = {}
 128     self._cache = Cache(cache_dir)
 129
 130   def Builders(self):
 131     return self._builders.values()
 132
 133   def Update(self):
 134     self._cache.Update("builders", self._root_url)
 135     self.FetchInfo()
 136
 137   def FetchInfo(self):
 138     if self._top_revision:
 139       return
 140
 141     html = self._cache.FetchData("builders", self._root_url)
 142
 143     """ Search for both builders and latest build number in HTML
 144     <td class="box"><a href="builders/<builder-name>"> identifies a builder
 145     <a href="builders/<builder-name>/builds/<build-num>"> is the latest build.
 146     """
 147     box_matcher = re.compile('.*a href[^>]*>([^<]*)\<')
 148     build_matcher = re.compile('.*a href=\"builders/(.*)/builds/([0-9]+)\".*')
 149     last_builder = ""
 150     for line in html:
 151       if 'a href="builders/' in line:
 152         if 'td class="box"' in line:
 153           last_builder = box_matcher.match(line).group(1)
 154           self._builders[last_builder] = Builder(self, last_builder)
 155         else:
 156           result = build_matcher.match(line)
 157           builder = result.group(1)
 158           assert builder == urllib.quote(last_builder)
 159           self._top_revision[last_builder] = int(result.group(2))
 160
 161   def GetLatestBuild(self, name):
 162     self.FetchInfo()
 163     assert self._top_revision
 164     return self._top_revision[name]
 165
 166
 167 class MultiLineChange(object):
 168   def __init__(self, lines):
 169     self._tracked_lines = lines
 170     self._current = 0
 171
 172   def __call__(self, line):
 173     """ Test a single line against multi-line change.
 174
 175     If it matches the currently active line, advance one line.
 176     If the current line is the last line, report a match.
 177     """
 178     if self._tracked_lines[self._current] in line:
 179       self._current = self._current + 1
 180       if self._current == len(self._tracked_lines):
 181         self._current = 0
 182         return True
 183     else:
 184       self._current = 0
 185     return False
 186
 187
 188 def main(argv):
 189   # Create argument parser.
 190   parser = argparse.ArgumentParser()
 191   commands = parser.add_mutually_exclusive_group(required=True)
 192   commands.add_argument("--update", action='store_true')
 193   commands.add_argument("--find", metavar='search term')
 194   args = parser.parse_args()
 195
 196   path = os.path.abspath(os.path.dirname(argv[0]))
 197   cache_path = os.path.join(path, CACHE_DIR)
 198
 199   fyi = Waterfall(ROOT_URL, cache_path)
 200
 201   if args.update:
 202     fyi.Update()
 203     for builder in fyi.Builders():
 204       print "Updating", builder.Name()
 205       builder.ScanLogs(lambda x:False)
 206
 207   if args.find:
 208     tester = MultiLineChange(args.find.splitlines())
 209     fyi.FetchInfo()
 210
 211     print "SCANNING FOR ", args.find
 212     for builder in fyi.Builders():
 213       print "Scanning", builder.Name()
 214       occurrences = builder.ScanLogs(tester)
 215       if occurrences:
 216         min_build = min(occurrences)
 217         path = builder.GetBuildPath(min_build)
 218         print "Earliest occurrence in build %d" % min_build
 219         print "Latest occurrence in build %d" % max(occurrences)
 220         print "Latest build: %d" % builder.LatestBuild()
 221         print path
 222         print "%d total" % len(occurrences)
 223
 224
 225 if __name__ == "__main__":
 226   sys.exit(main(sys.argv))
 227