tools/bisect-builds.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Snapshot Build Bisect Tool
   7
   8 This script bisects a snapshot archive using binary search. It starts at
   9 a bad revision (it will try to guess HEAD) and asks for a last known-good
  10 revision. It will then binary search across this revision range by downloading,
  11 unzipping, and opening Chromium for you. After testing the specific revision,
  12 it will ask you whether it is good or bad before continuing the search.
  13 """
  14
  15 # The root URL for storage.
  16 CHROMIUM_BASE_URL = 'http://commondatastorage.googleapis.com/chromium-browser-snapshots'
  17 WEBKIT_BASE_URL = 'http://commondatastorage.googleapis.com/chromium-webkit-snapshots'
  18
  19 # The root URL for official builds.
  20 OFFICIAL_BASE_URL = 'http://master.chrome.corp.google.com/official_builds'
  21
  22 # Changelogs URL.
  23 CHANGELOG_URL = 'http://build.chromium.org/f/chromium/' \
  24                 'perf/dashboard/ui/changelog.html?' \
  25                 'url=/trunk/src&range=%d%%3A%d'
  26
  27 # Official Changelogs URL.
  28 OFFICIAL_CHANGELOG_URL = 'http://omahaproxy.appspot.com/'\
  29                          'changelog?old_version=%s&new_version=%s'
  30
  31 # DEPS file URL.
  32 DEPS_FILE = 'http://src.chromium.org/viewvc/chrome/trunk/src/DEPS?revision=%d'
  33 # Blink Changelogs URL.
  34 BLINK_CHANGELOG_URL = 'http://build.chromium.org/f/chromium/' \
  35                       'perf/dashboard/ui/changelog_blink.html?' \
  36                       'url=/trunk&range=%d%%3A%d'
  37
  38 DONE_MESSAGE_GOOD_MIN = 'You are probably looking for a change made after %s ' \
  39                         '(known good), but no later than %s (first known bad).'
  40 DONE_MESSAGE_GOOD_MAX = 'You are probably looking for a change made after %s ' \
  41                         '(known bad), but no later than %s (first known good).'
  42
  43 ###############################################################################
  44
  45 import json
  46 import optparse
  47 import os
  48 import re
  49 import shlex
  50 import shutil
  51 import subprocess
  52 import sys
  53 import tempfile
  54 import threading
  55 import urllib
  56 from distutils.version import LooseVersion
  57 from xml.etree import ElementTree
  58 import zipfile
  59
  60
  61 class PathContext(object):
  62   """A PathContext is used to carry the information used to construct URLs and
  63   paths when dealing with the storage server and archives."""
  64   def __init__(self, base_url, platform, good_revision, bad_revision,
  65                is_official, is_aura, flash_path = None, pdf_path = None):
  66     super(PathContext, self).__init__()
  67     # Store off the input parameters.
  68     self.base_url = base_url
  69     self.platform = platform  # What's passed in to the '-a/--archive' option.
  70     self.good_revision = good_revision
  71     self.bad_revision = bad_revision
  72     self.is_official = is_official
  73     self.is_aura = is_aura
  74     self.flash_path = flash_path
  75     self.pdf_path = pdf_path
  76
  77     # The name of the ZIP file in a revision directory on the server.
  78     self.archive_name = None
  79
  80     # Set some internal members:
  81     #   _listing_platform_dir = Directory that holds revisions. Ends with a '/'.
  82     #   _archive_extract_dir = Uncompressed directory in the archive_name file.
  83     #   _binary_name = The name of the executable to run.
  84     if self.platform in ('linux', 'linux64', 'linux-arm'):
  85       self._binary_name = 'chrome'
  86     elif self.platform == 'mac':
  87       self.archive_name = 'chrome-mac.zip'
  88       self._archive_extract_dir = 'chrome-mac'
  89     elif self.platform == 'win':
  90       self.archive_name = 'chrome-win32.zip'
  91       self._archive_extract_dir = 'chrome-win32'
  92       self._binary_name = 'chrome.exe'
  93     else:
  94       raise Exception('Invalid platform: %s' % self.platform)
  95
  96     if is_official:
  97       if self.platform == 'linux':
  98         self._listing_platform_dir = 'precise32bit/'
  99         self.archive_name = 'chrome-precise32bit.zip'
 100         self._archive_extract_dir = 'chrome-precise32bit'
 101       elif self.platform == 'linux64':
 102         self._listing_platform_dir = 'precise64bit/'
 103         self.archive_name = 'chrome-precise64bit.zip'
 104         self._archive_extract_dir = 'chrome-precise64bit'
 105       elif self.platform == 'mac':
 106         self._listing_platform_dir = 'mac/'
 107         self._binary_name = 'Google Chrome.app/Contents/MacOS/Google Chrome'
 108       elif self.platform == 'win':
 109         if self.is_aura:
 110           self._listing_platform_dir = 'win-aura/'
 111         else:
 112           self._listing_platform_dir = 'win/'
 113     else:
 114       if self.platform in ('linux', 'linux64', 'linux-arm'):
 115         self.archive_name = 'chrome-linux.zip'
 116         self._archive_extract_dir = 'chrome-linux'
 117         if self.platform == 'linux':
 118           self._listing_platform_dir = 'Linux/'
 119         elif self.platform == 'linux64':
 120           self._listing_platform_dir = 'Linux_x64/'
 121         elif self.platform == 'linux-arm':
 122           self._listing_platform_dir = 'Linux_ARM_Cross-Compile/'
 123       elif self.platform == 'mac':
 124         self._listing_platform_dir = 'Mac/'
 125         self._binary_name = 'Chromium.app/Contents/MacOS/Chromium'
 126       elif self.platform == 'win':
 127         self._listing_platform_dir = 'Win/'
 128
 129   def GetListingURL(self, marker=None):
 130     """Returns the URL for a directory listing, with an optional marker."""
 131     marker_param = ''
 132     if marker:
 133       marker_param = '&marker=' + str(marker)
 134     return self.base_url + '/?delimiter=/&prefix=' + \
 135         self._listing_platform_dir + marker_param
 136
 137   def GetDownloadURL(self, revision):
 138     """Gets the download URL for a build archive of a specific revision."""
 139     if self.is_official:
 140       return "%s/%s/%s%s" % (
 141           OFFICIAL_BASE_URL, revision, self._listing_platform_dir,
 142           self.archive_name)
 143     else:
 144       return "%s/%s%s/%s" % (self.base_url, self._listing_platform_dir,
 145                              revision, self.archive_name)
 146
 147   def GetLastChangeURL(self):
 148     """Returns a URL to the LAST_CHANGE file."""
 149     return self.base_url + '/' + self._listing_platform_dir + 'LAST_CHANGE'
 150
 151   def GetLaunchPath(self):
 152     """Returns a relative path (presumably from the archive extraction location)
 153     that is used to run the executable."""
 154     return os.path.join(self._archive_extract_dir, self._binary_name)
 155
 156   def IsAuraBuild(self, build):
 157     """Check the given build is Aura."""
 158     return build.split('.')[3] == '1'
 159
 160   def IsASANBuild(self, build):
 161     """Check the given build is ASAN build."""
 162     return build.split('.')[3] == '2'
 163
 164   def ParseDirectoryIndex(self):
 165     """Parses the Google Storage directory listing into a list of revision
 166     numbers."""
 167
 168     def _FetchAndParse(url):
 169       """Fetches a URL and returns a 2-Tuple of ([revisions], next-marker). If
 170       next-marker is not None, then the listing is a partial listing and another
 171       fetch should be performed with next-marker being the marker= GET
 172       parameter."""
 173       handle = urllib.urlopen(url)
 174       document = ElementTree.parse(handle)
 175
 176       # All nodes in the tree are namespaced. Get the root's tag name to extract
 177       # the namespace. Etree does namespaces as |{namespace}tag|.
 178       root_tag = document.getroot().tag
 179       end_ns_pos = root_tag.find('}')
 180       if end_ns_pos == -1:
 181         raise Exception("Could not locate end namespace for directory index")
 182       namespace = root_tag[:end_ns_pos + 1]
 183
 184       # Find the prefix (_listing_platform_dir) and whether or not the list is
 185       # truncated.
 186       prefix_len = len(document.find(namespace + 'Prefix').text)
 187       next_marker = None
 188       is_truncated = document.find(namespace + 'IsTruncated')
 189       if is_truncated is not None and is_truncated.text.lower() == 'true':
 190         next_marker = document.find(namespace + 'NextMarker').text
 191
 192       # Get a list of all the revisions.
 193       all_prefixes = document.findall(namespace + 'CommonPrefixes/' +
 194                                       namespace + 'Prefix')
 195       # The <Prefix> nodes have content of the form of
 196       # |_listing_platform_dir/revision/|. Strip off the platform dir and the
 197       # trailing slash to just have a number.
 198       revisions = []
 199       for prefix in all_prefixes:
 200         revnum = prefix.text[prefix_len:-1]
 201         try:
 202           revnum = int(revnum)
 203           revisions.append(revnum)
 204         except ValueError:
 205           pass
 206       return (revisions, next_marker)
 207
 208     # Fetch the first list of revisions.
 209     (revisions, next_marker) = _FetchAndParse(self.GetListingURL())
 210
 211     # If the result list was truncated, refetch with the next marker. Do this
 212     # until an entire directory listing is done.
 213     while next_marker:
 214       next_url = self.GetListingURL(next_marker)
 215       (new_revisions, next_marker) = _FetchAndParse(next_url)
 216       revisions.extend(new_revisions)
 217     return revisions
 218
 219   def GetRevList(self):
 220     """Gets the list of revision numbers between self.good_revision and
 221     self.bad_revision."""
 222     # Download the revlist and filter for just the range between good and bad.
 223     minrev = min(self.good_revision, self.bad_revision)
 224     maxrev = max(self.good_revision, self.bad_revision)
 225     revlist_all = map(int, self.ParseDirectoryIndex())
 226
 227     revlist = [x for x in revlist_all if x >= int(minrev) and x <= int(maxrev)]
 228     revlist.sort()
 229
 230     # Set good and bad revisions to be legit revisions.
 231     if revlist:
 232       if self.good_revision < self.bad_revision:
 233         self.good_revision = revlist[0]
 234         self.bad_revision = revlist[-1]
 235       else:
 236         self.bad_revision = revlist[0]
 237         self.good_revision = revlist[-1]
 238
 239       # Fix chromium rev so that the deps blink revision matches REVISIONS file.
 240       if self.base_url == WEBKIT_BASE_URL:
 241         revlist_all.sort()
 242         self.good_revision = FixChromiumRevForBlink(revlist,
 243                                                     revlist_all,
 244                                                     self,
 245                                                     self.good_revision)
 246         self.bad_revision = FixChromiumRevForBlink(revlist,
 247                                                    revlist_all,
 248                                                    self,
 249                                                    self.bad_revision)
 250     return revlist
 251
 252   def GetOfficialBuildsList(self):
 253     """Gets the list of official build numbers between self.good_revision and
 254     self.bad_revision."""
 255     # Download the revlist and filter for just the range between good and bad.
 256     minrev = min(self.good_revision, self.bad_revision)
 257     maxrev = max(self.good_revision, self.bad_revision)
 258     handle = urllib.urlopen(OFFICIAL_BASE_URL)
 259     dirindex = handle.read()
 260     handle.close()
 261     build_numbers = re.findall(r'<a href="([0-9][0-9].*)/">', dirindex)
 262     final_list = []
 263     i = 0
 264     parsed_build_numbers = [LooseVersion(x) for x in build_numbers]
 265     for build_number in sorted(parsed_build_numbers):
 266       path = OFFICIAL_BASE_URL + '/' + str(build_number) + '/' + \
 267              self._listing_platform_dir + self.archive_name
 268       i = i + 1
 269       try:
 270         connection = urllib.urlopen(path)
 271         connection.close()
 272         if build_number > maxrev:
 273           break
 274         if build_number >= minrev:
 275           # If we are bisecting Aura, we want to include only builds which
 276           # ends with ".1".
 277           if self.is_aura:
 278             if self.IsAuraBuild(str(build_number)):
 279               final_list.append(str(build_number))
 280           # If we are bisecting only official builds (without --aura),
 281           # we can not include builds which ends with '.1' or '.2' since
 282           # they have different folder hierarchy inside.
 283           elif (not self.IsAuraBuild(str(build_number)) and
 284                 not self.IsASANBuild(str(build_number))):
 285             final_list.append(str(build_number))
 286       except urllib.HTTPError, e:
 287         pass
 288     return final_list
 289
 290 def UnzipFilenameToDir(filename, directory):
 291   """Unzip |filename| to |directory|."""
 292   cwd = os.getcwd()
 293   if not os.path.isabs(filename):
 294     filename = os.path.join(cwd, filename)
 295   zf = zipfile.ZipFile(filename)
 296   # Make base.
 297   if not os.path.isdir(directory):
 298     os.mkdir(directory)
 299   os.chdir(directory)
 300   # Extract files.
 301   for info in zf.infolist():
 302     name = info.filename
 303     if name.endswith('/'):  # dir
 304       if not os.path.isdir(name):
 305         os.makedirs(name)
 306     else:  # file
 307       directory = os.path.dirname(name)
 308       if not os.path.isdir(directory):
 309         os.makedirs(directory)
 310       out = open(name, 'wb')
 311       out.write(zf.read(name))
 312       out.close()
 313     # Set permissions. Permission info in external_attr is shifted 16 bits.
 314     os.chmod(name, info.external_attr >> 16L)
 315   os.chdir(cwd)
 316
 317
 318 def FetchRevision(context, rev, filename, quit_event=None, progress_event=None):
 319   """Downloads and unzips revision |rev|.
 320   @param context A PathContext instance.
 321   @param rev The Chromium revision number/tag to download.
 322   @param filename The destination for the downloaded file.
 323   @param quit_event A threading.Event which will be set by the master thread to
 324                     indicate that the download should be aborted.
 325   @param progress_event A threading.Event which will be set by the master thread
 326                     to indicate that the progress of the download should be
 327                     displayed.
 328   """
 329   def ReportHook(blocknum, blocksize, totalsize):
 330     if quit_event and quit_event.isSet():
 331       raise RuntimeError("Aborting download of revision %s" % str(rev))
 332     if progress_event and progress_event.isSet():
 333       size = blocknum * blocksize
 334       if totalsize == -1:  # Total size not known.
 335         progress = "Received %d bytes" % size
 336       else:
 337         size = min(totalsize, size)
 338         progress = "Received %d of %d bytes, %.2f%%" % (
 339             size, totalsize, 100.0 * size / totalsize)
 340       # Send a \r to let all progress messages use just one line of output.
 341       sys.stdout.write("\r" + progress)
 342       sys.stdout.flush()
 343
 344   download_url = context.GetDownloadURL(rev)
 345   try:
 346     urllib.urlretrieve(download_url, filename, ReportHook)
 347     if progress_event and progress_event.isSet():
 348       print
 349   except RuntimeError, e:
 350     pass
 351
 352
 353 def RunRevision(context, revision, zipfile, profile, num_runs, command, args):
 354   """Given a zipped revision, unzip it and run the test."""
 355   print "Trying revision %s..." % str(revision)
 356
 357   # Create a temp directory and unzip the revision into it.
 358   cwd = os.getcwd()
 359   tempdir = tempfile.mkdtemp(prefix='bisect_tmp')
 360   UnzipFilenameToDir(zipfile, tempdir)
 361   os.chdir(tempdir)
 362
 363   # Run the build as many times as specified.
 364   testargs = ['--user-data-dir=%s' % profile] + args
 365   # The sandbox must be run as root on Official Chrome, so bypass it.
 366   if ((context.is_official or context.flash_path or context.pdf_path) and
 367       context.platform.startswith('linux')):
 368     testargs.append('--no-sandbox')
 369   if context.flash_path:
 370     testargs.append('--ppapi-flash-path=%s' % context.flash_path)
 371     # We have to pass a large enough Flash version, which currently needs not
 372     # be correct. Instead of requiring the user of the script to figure out and
 373     # pass the correct version we just spoof it.
 374     testargs.append('--ppapi-flash-version=99.9.999.999')
 375
 376   if context.pdf_path:
 377     shutil.copy(context.pdf_path, os.path.dirname(context.GetLaunchPath()))
 378     testargs.append('--enable-print-preview')
 379
 380   runcommand = []
 381   for token in shlex.split(command):
 382     if token == "%a":
 383       runcommand.extend(testargs)
 384     else:
 385       runcommand.append( \
 386           token.replace('%p', context.GetLaunchPath()) \
 387                .replace('%s', ' '.join(testargs)))
 388
 389   for i in range(0, num_runs):
 390     subproc = subprocess.Popen(runcommand,
 391                                bufsize=-1,
 392                                stdout=subprocess.PIPE,
 393                                stderr=subprocess.PIPE)
 394     (stdout, stderr) = subproc.communicate()
 395
 396   os.chdir(cwd)
 397   try:
 398     shutil.rmtree(tempdir, True)
 399   except Exception, e:
 400     pass
 401
 402   return (subproc.returncode, stdout, stderr)
 403
 404
 405 def AskIsGoodBuild(rev, official_builds, status, stdout, stderr):
 406   """Ask the user whether build |rev| is good or bad."""
 407   # Loop until we get a response that we can parse.
 408   while True:
 409     response = raw_input('Revision %s is ' \
 410                          '[(g)ood/(b)ad/(r)etry/(u)nknown/(q)uit]: ' %
 411                          str(rev))
 412     if response and response in ('g', 'b', 'r', 'u'):
 413       return response
 414     if response and response == 'q':
 415       raise SystemExit()
 416
 417
 418 class DownloadJob(object):
 419   """DownloadJob represents a task to download a given Chromium revision."""
 420   def __init__(self, context, name, rev, zipfile):
 421     super(DownloadJob, self).__init__()
 422     # Store off the input parameters.
 423     self.context = context
 424     self.name = name
 425     self.rev = rev
 426     self.zipfile = zipfile
 427     self.quit_event = threading.Event()
 428     self.progress_event = threading.Event()
 429
 430   def Start(self):
 431     """Starts the download."""
 432     fetchargs = (self.context,
 433                  self.rev,
 434                  self.zipfile,
 435                  self.quit_event,
 436                  self.progress_event)
 437     self.thread = threading.Thread(target=FetchRevision,
 438                                    name=self.name,
 439                                    args=fetchargs)
 440     self.thread.start()
 441
 442   def Stop(self):
 443     """Stops the download which must have been started previously."""
 444     self.quit_event.set()
 445     self.thread.join()
 446     os.unlink(self.zipfile)
 447
 448   def WaitFor(self):
 449     """Prints a message and waits for the download to complete. The download
 450     must have been started previously."""
 451     print "Downloading revision %s..." % str(self.rev)
 452     self.progress_event.set()  # Display progress of download.
 453     self.thread.join()
 454
 455
 456 def Bisect(base_url,
 457            platform,
 458            official_builds,
 459            is_aura,
 460            good_rev=0,
 461            bad_rev=0,
 462            num_runs=1,
 463            command="%p %a",
 464            try_args=(),
 465            profile=None,
 466            flash_path=None,
 467            pdf_path=None,
 468            evaluate=AskIsGoodBuild):
 469   """Given known good and known bad revisions, run a binary search on all
 470   archived revisions to determine the last known good revision.
 471
 472   @param platform Which build to download/run ('mac', 'win', 'linux64', etc.).
 473   @param official_builds Specify build type (Chromium or Official build).
 474   @param good_rev Number/tag of the known good revision.
 475   @param bad_rev Number/tag of the known bad revision.
 476   @param num_runs Number of times to run each build for asking good/bad.
 477   @param try_args A tuple of arguments to pass to the test application.
 478   @param profile The name of the user profile to run with.
 479   @param evaluate A function which returns 'g' if the argument build is good,
 480                   'b' if it's bad or 'u' if unknown.
 481
 482   Threading is used to fetch Chromium revisions in the background, speeding up
 483   the user's experience. For example, suppose the bounds of the search are
 484   good_rev=0, bad_rev=100. The first revision to be checked is 50. Depending on
 485   whether revision 50 is good or bad, the next revision to check will be either
 486   25 or 75. So, while revision 50 is being checked, the script will download
 487   revisions 25 and 75 in the background. Once the good/bad verdict on rev 50 is
 488   known:
 489
 490     - If rev 50 is good, the download of rev 25 is cancelled, and the next test
 491       is run on rev 75.
 492
 493     - If rev 50 is bad, the download of rev 75 is cancelled, and the next test
 494       is run on rev 25.
 495   """
 496
 497   if not profile:
 498     profile = 'profile'
 499
 500   context = PathContext(base_url, platform, good_rev, bad_rev,
 501                         official_builds, is_aura, flash_path, pdf_path)
 502   cwd = os.getcwd()
 503
 504   print "Downloading list of known revisions..."
 505   _GetDownloadPath = lambda rev: os.path.join(cwd,
 506       '%s-%s' % (str(rev), context.archive_name))
 507   if official_builds:
 508     revlist = context.GetOfficialBuildsList()
 509   else:
 510     revlist = context.GetRevList()
 511
 512   # Get a list of revisions to bisect across.
 513   if len(revlist) < 2:  # Don't have enough builds to bisect.
 514     msg = 'We don\'t have enough builds to bisect. revlist: %s' % revlist
 515     raise RuntimeError(msg)
 516
 517   # Figure out our bookends and first pivot point; fetch the pivot revision.
 518   minrev = 0
 519   maxrev = len(revlist) - 1
 520   pivot = maxrev / 2
 521   rev = revlist[pivot]
 522   zipfile = _GetDownloadPath(rev)
 523   fetch = DownloadJob(context, 'initial_fetch', rev, zipfile)
 524   fetch.Start()
 525   fetch.WaitFor()
 526
 527   # Binary search time!
 528   while fetch and fetch.zipfile and maxrev - minrev > 1:
 529     if bad_rev < good_rev:
 530       min_str, max_str = "bad", "good"
 531     else:
 532       min_str, max_str = "good", "bad"
 533     print 'Bisecting range [%s (%s), %s (%s)].' % (revlist[minrev], min_str, \
 534                                                    revlist[maxrev], max_str)
 535
 536     # Pre-fetch next two possible pivots
 537     #   - down_pivot is the next revision to check if the current revision turns
 538     #     out to be bad.
 539     #   - up_pivot is the next revision to check if the current revision turns
 540     #     out to be good.
 541     down_pivot = int((pivot - minrev) / 2) + minrev
 542     down_fetch = None
 543     if down_pivot != pivot and down_pivot != minrev:
 544       down_rev = revlist[down_pivot]
 545       down_fetch = DownloadJob(context, 'down_fetch', down_rev,
 546                                _GetDownloadPath(down_rev))
 547       down_fetch.Start()
 548
 549     up_pivot = int((maxrev - pivot) / 2) + pivot
 550     up_fetch = None
 551     if up_pivot != pivot and up_pivot != maxrev:
 552       up_rev = revlist[up_pivot]
 553       up_fetch = DownloadJob(context, 'up_fetch', up_rev,
 554                              _GetDownloadPath(up_rev))
 555       up_fetch.Start()
 556
 557     # Run test on the pivot revision.
 558     status = None
 559     stdout = None
 560     stderr = None
 561     try:
 562       (status, stdout, stderr) = RunRevision(context,
 563                                              rev,
 564                                              fetch.zipfile,
 565                                              profile,
 566                                              num_runs,
 567                                              command,
 568                                              try_args)
 569     except Exception, e:
 570       print >> sys.stderr, e
 571
 572     # Call the evaluate function to see if the current revision is good or bad.
 573     # On that basis, kill one of the background downloads and complete the
 574     # other, as described in the comments above.
 575     try:
 576       answer = evaluate(rev, official_builds, status, stdout, stderr)
 577       if answer == 'g' and good_rev < bad_rev or \
 578           answer == 'b' and bad_rev < good_rev:
 579         fetch.Stop()
 580         minrev = pivot
 581         if down_fetch:
 582           down_fetch.Stop()  # Kill the download of the older revision.
 583           fetch = None
 584         if up_fetch:
 585           up_fetch.WaitFor()
 586           pivot = up_pivot
 587           fetch = up_fetch
 588       elif answer == 'b' and good_rev < bad_rev or \
 589           answer == 'g' and bad_rev < good_rev:
 590         fetch.Stop()
 591         maxrev = pivot
 592         if up_fetch:
 593           up_fetch.Stop()  # Kill the download of the newer revision.
 594           fetch = None
 595         if down_fetch:
 596           down_fetch.WaitFor()
 597           pivot = down_pivot
 598           fetch = down_fetch
 599       elif answer == 'r':
 600         pass  # Retry requires no changes.
 601       elif answer == 'u':
 602         # Nuke the revision from the revlist and choose a new pivot.
 603         fetch.Stop()
 604         revlist.pop(pivot)
 605         maxrev -= 1  # Assumes maxrev >= pivot.
 606
 607         if maxrev - minrev > 1:
 608           # Alternate between using down_pivot or up_pivot for the new pivot
 609           # point, without affecting the range. Do this instead of setting the
 610           # pivot to the midpoint of the new range because adjacent revisions
 611           # are likely affected by the same issue that caused the (u)nknown
 612           # response.
 613           if up_fetch and down_fetch:
 614             fetch = [up_fetch, down_fetch][len(revlist) % 2]
 615           elif up_fetch:
 616             fetch = up_fetch
 617           else:
 618             fetch = down_fetch
 619           fetch.WaitFor()
 620           if fetch == up_fetch:
 621             pivot = up_pivot - 1  # Subtracts 1 because revlist was resized.
 622           else:
 623             pivot = down_pivot
 624           zipfile = fetch.zipfile
 625
 626         if down_fetch and fetch != down_fetch:
 627           down_fetch.Stop()
 628         if up_fetch and fetch != up_fetch:
 629           up_fetch.Stop()
 630       else:
 631         assert False, "Unexpected return value from evaluate(): " + answer
 632     except SystemExit:
 633       print "Cleaning up..."
 634       for f in [_GetDownloadPath(revlist[down_pivot]),
 635                 _GetDownloadPath(revlist[up_pivot])]:
 636         try:
 637           os.unlink(f)
 638         except OSError:
 639           pass
 640       sys.exit(0)
 641
 642     rev = revlist[pivot]
 643
 644   return (revlist[minrev], revlist[maxrev])
 645
 646
 647 def GetBlinkDEPSRevisionForChromiumRevision(rev):
 648   """Returns the blink revision that was in REVISIONS file at
 649   chromium revision |rev|."""
 650   # . doesn't match newlines without re.DOTALL, so this is safe.
 651   blink_re = re.compile(r'webkit_revision\D*(\d+)')
 652   url = urllib.urlopen(DEPS_FILE % rev)
 653   m = blink_re.search(url.read())
 654   url.close()
 655   if m:
 656     return int(m.group(1))
 657   else:
 658     raise Exception('Could not get Blink revision for Chromium rev %d'
 659                     % rev)
 660
 661
 662 def GetBlinkRevisionForChromiumRevision(self, rev):
 663   """Returns the blink revision that was in REVISIONS file at
 664   chromium revision |rev|."""
 665   file_url = "%s/%s%d/REVISIONS" % (self.base_url,
 666                                     self._listing_platform_dir, rev)
 667   url = urllib.urlopen(file_url)
 668   data = json.loads(url.read())
 669   url.close()
 670   if 'webkit_revision' in data:
 671     return data['webkit_revision']
 672   else:
 673     raise Exception('Could not get blink revision for cr rev %d' % rev)
 674
 675 def FixChromiumRevForBlink(revisions_final, revisions, self, rev):
 676   """Returns the chromium revision that has the correct blink revision
 677   for blink bisect, DEPS and REVISIONS file might not match since
 678   blink snapshots point to tip of tree blink.
 679   Note: The revisions_final variable might get modified to include
 680   additional revisions."""
 681
 682   blink_deps_rev = GetBlinkDEPSRevisionForChromiumRevision(rev)
 683
 684   while (GetBlinkRevisionForChromiumRevision(self, rev) > blink_deps_rev):
 685     idx = revisions.index(rev)
 686     if idx > 0:
 687       rev = revisions[idx-1]
 688       if rev not in revisions_final:
 689         revisions_final.insert(0, rev)
 690
 691   revisions_final.sort()
 692   return rev
 693
 694 def GetChromiumRevision(url):
 695   """Returns the chromium revision read from given URL."""
 696   try:
 697     # Location of the latest build revision number
 698     return int(urllib.urlopen(url).read())
 699   except Exception, e:
 700     print('Could not determine latest revision. This could be bad...')
 701     return 999999999
 702
 703
 704 def main():
 705   usage = ('%prog [options] [-- chromium-options]\n'
 706            'Perform binary search on the snapshot builds to find a minimal\n'
 707            'range of revisions where a behavior change happened. The\n'
 708            'behaviors are described as "good" and "bad".\n'
 709            'It is NOT assumed that the behavior of the later revision is\n'
 710            'the bad one.\n'
 711            '\n'
 712            'Revision numbers should use\n'
 713            '  Official versions (e.g. 1.0.1000.0) for official builds. (-o)\n'
 714            '  SVN revisions (e.g. 123456) for chromium builds, from trunk.\n'
 715            '    Use base_trunk_revision from http://omahaproxy.appspot.com/\n'
 716            '    for earlier revs.\n'
 717            '    Chrome\'s about: build number and omahaproxy branch_revision\n'
 718            '    are incorrect, they are from branches.\n'
 719            '\n'
 720            'Tip: add "-- --no-first-run" to bypass the first run prompts.')
 721   parser = optparse.OptionParser(usage=usage)
 722   # Strangely, the default help output doesn't include the choice list.
 723   choices = ['mac', 'win', 'linux', 'linux64', 'linux-arm']
 724             # linux-chromiumos lacks a continuous archive http://crbug.com/78158
 725   parser.add_option('-a', '--archive',
 726                     choices = choices,
 727                     help = 'The buildbot archive to bisect [%s].' %
 728                            '|'.join(choices))
 729   parser.add_option('-o', action="store_true", dest='official_builds',
 730                     help = 'Bisect across official ' +
 731                     'Chrome builds (internal only) instead of ' +
 732                     'Chromium archives.')
 733   parser.add_option('-b', '--bad', type = 'str',
 734                     help = 'A bad revision to start bisection. ' +
 735                     'May be earlier or later than the good revision. ' +
 736                     'Default is HEAD.')
 737   parser.add_option('-f', '--flash_path', type = 'str',
 738                     help = 'Absolute path to a recent Adobe Pepper Flash ' +
 739                     'binary to be used in this bisection (e.g. ' +
 740                     'on Windows C:\...\pepflashplayer.dll and on Linux ' +
 741                     '/opt/google/chrome/PepperFlash/libpepflashplayer.so).')
 742   parser.add_option('-d', '--pdf_path', type = 'str',
 743                     help = 'Absolute path to a recent PDF pluggin ' +
 744                     'binary to be used in this bisection (e.g. ' +
 745                     'on Windows C:\...\pdf.dll and on Linux ' +
 746                     '/opt/google/chrome/libpdf.so). Option also enables ' +
 747                     'print preview.')
 748   parser.add_option('-g', '--good', type = 'str',
 749                     help = 'A good revision to start bisection. ' +
 750                     'May be earlier or later than the bad revision. ' +
 751                     'Default is 0.')
 752   parser.add_option('-p', '--profile', '--user-data-dir', type = 'str',
 753                     help = 'Profile to use; this will not reset every run. ' +
 754                     'Defaults to a clean profile.', default = 'profile')
 755   parser.add_option('-t', '--times', type = 'int',
 756                     help = 'Number of times to run each build before asking ' +
 757                     'if it\'s good or bad. Temporary profiles are reused.',
 758                     default = 1)
 759   parser.add_option('-c', '--command', type = 'str',
 760                     help = 'Command to execute. %p and %a refer to Chrome ' +
 761                     'executable and specified extra arguments respectively. ' +
 762                     'Use %s to specify all extra arguments as one string. ' +
 763                     'Defaults to "%p %a". Note that any extra paths ' +
 764                     'specified should be absolute.',
 765                     default = '%p %a')
 766   parser.add_option('-l', '--blink', action='store_true',
 767                     help = 'Use Blink bisect instead of Chromium. ')
 768   parser.add_option('--aura',
 769                     dest='aura',
 770                     action='store_true',
 771                     default=False,
 772                     help='Allow the script to bisect aura builds')
 773
 774   (opts, args) = parser.parse_args()
 775
 776   if opts.archive is None:
 777     print 'Error: missing required parameter: --archive'
 778     print
 779     parser.print_help()
 780     return 1
 781
 782   if opts.aura:
 783     if opts.archive != 'win' or not opts.official_builds:
 784       print 'Error: Aura is supported only on Windows platform '\
 785             'and official builds.'
 786       return 1
 787
 788   if opts.blink:
 789     base_url = WEBKIT_BASE_URL
 790   else:
 791     base_url = CHROMIUM_BASE_URL
 792
 793   # Create the context. Initialize 0 for the revisions as they are set below.
 794   context = PathContext(base_url, opts.archive, 0, 0,
 795                         opts.official_builds, opts.aura, None)
 796   # Pick a starting point, try to get HEAD for this.
 797   if opts.bad:
 798     bad_rev = opts.bad
 799   else:
 800     bad_rev = '999.0.0.0'
 801     if not opts.official_builds:
 802       bad_rev = GetChromiumRevision(context.GetLastChangeURL())
 803
 804   # Find out when we were good.
 805   if opts.good:
 806     good_rev = opts.good
 807   else:
 808     good_rev = '0.0.0.0' if opts.official_builds else 0
 809
 810   if opts.flash_path:
 811     flash_path = opts.flash_path
 812     msg = 'Could not find Flash binary at %s' % flash_path
 813     assert os.path.exists(flash_path), msg
 814
 815   if opts.pdf_path:
 816     pdf_path = opts.pdf_path
 817     msg = 'Could not find PDF binary at %s' % pdf_path
 818     assert os.path.exists(pdf_path), msg
 819
 820   if opts.official_builds:
 821     good_rev = LooseVersion(good_rev)
 822     bad_rev = LooseVersion(bad_rev)
 823   else:
 824     good_rev = int(good_rev)
 825     bad_rev = int(bad_rev)
 826
 827   if opts.times < 1:
 828     print('Number of times to run (%d) must be greater than or equal to 1.' %
 829           opts.times)
 830     parser.print_help()
 831     return 1
 832
 833   (min_chromium_rev, max_chromium_rev) = Bisect(
 834       base_url, opts.archive, opts.official_builds, opts.aura, good_rev,
 835       bad_rev, opts.times, opts.command, args, opts.profile, opts.flash_path,
 836       opts.pdf_path)
 837
 838   # Get corresponding blink revisions.
 839   try:
 840     min_blink_rev = GetBlinkRevisionForChromiumRevision(context,
 841                                                         min_chromium_rev)
 842     max_blink_rev = GetBlinkRevisionForChromiumRevision(context,
 843                                                         max_chromium_rev)
 844   except Exception, e:
 845     # Silently ignore the failure.
 846     min_blink_rev, max_blink_rev = 0, 0
 847
 848   if opts.blink:
 849     # We're done. Let the user know the results in an official manner.
 850     if good_rev > bad_rev:
 851       print DONE_MESSAGE_GOOD_MAX % (str(min_blink_rev), str(max_blink_rev))
 852     else:
 853       print DONE_MESSAGE_GOOD_MIN % (str(min_blink_rev), str(max_blink_rev))
 854
 855     print 'BLINK CHANGELOG URL:'
 856     print '  ' + BLINK_CHANGELOG_URL % (max_blink_rev, min_blink_rev)
 857
 858   else:
 859     # We're done. Let the user know the results in an official manner.
 860     if good_rev > bad_rev:
 861       print DONE_MESSAGE_GOOD_MAX % (str(min_chromium_rev),
 862                                      str(max_chromium_rev))
 863     else:
 864       print DONE_MESSAGE_GOOD_MIN % (str(min_chromium_rev),
 865                                      str(max_chromium_rev))
 866     if min_blink_rev != max_blink_rev:
 867       print ("NOTE: There is a Blink roll in the range, "
 868              "you might also want to do a Blink bisect.")
 869
 870     print 'CHANGELOG URL:'
 871     if opts.official_builds:
 872       print OFFICIAL_CHANGELOG_URL % (min_chromium_rev, max_chromium_rev)
 873     else:
 874       print '  ' + CHANGELOG_URL % (min_chromium_rev, max_chromium_rev)
 875
 876 if __name__ == '__main__':
 877   sys.exit(main())