tools/bisect_utils.py

   1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Set of operations/utilities related to checking out the depot, and
   6 outputting annotations on the buildbot waterfall. These are intended to be
   7 used by the bisection scripts."""
   8
   9 import errno
  10 import os
  11 import shutil
  12 import subprocess
  13 import sys
  14
  15 GCLIENT_SPEC_DATA = [
  16   { "name"        : "src",
  17     "url"         : "https://chromium.googlesource.com/chromium/src.git",
  18     "deps_file"   : ".DEPS.git",
  19     "managed"     : True,
  20     "custom_deps" : {
  21       "src/data/page_cycler": "https://chrome-internal.googlesource.com/"
  22                               "chrome/data/page_cycler/.git",
  23       "src/data/dom_perf": "https://chrome-internal.googlesource.com/"
  24                            "chrome/data/dom_perf/.git",
  25       "src/tools/perf/data": "https://chrome-internal.googlesource.com/"
  26                              "chrome/tools/perf/data/.git",
  27     },
  28     "safesync_url": "",
  29   },
  30 ]
  31 GCLIENT_ANDROID = "\ntarget_os = ['android']"
  32 GCLIENT_CUSTOM_DEPS_V8 = {"src/v8_bleeding_edge": "git://github.com/v8/v8.git"}
  33 FILE_DEPS_GIT = '.DEPS.git'
  34
  35 REPO_PARAMS = [
  36   'https://chrome-internal.googlesource.com/chromeos/manifest-internal/',
  37   '--repo-url',
  38   'https://git.chromium.org/external/repo.git'
  39 ]
  40
  41 REPO_SYNC_COMMAND = 'git checkout -f $(git rev-list --max-count=1 '\
  42                     '--before=%d remotes/m/master)'
  43
  44 def OutputAnnotationStepStart(name):
  45   """Outputs appropriate annotation to signal the start of a step to
  46   a trybot.
  47
  48   Args:
  49     name: The name of the step.
  50   """
  51   print
  52   print '@@@SEED_STEP %s@@@' % name
  53   print '@@@STEP_CURSOR %s@@@' % name
  54   print '@@@STEP_STARTED@@@'
  55   print
  56
  57
  58 def OutputAnnotationStepClosed():
  59   """Outputs appropriate annotation to signal the closing of a step to
  60   a trybot."""
  61   print
  62   print '@@@STEP_CLOSED@@@'
  63   print
  64
  65
  66 def CreateAndChangeToSourceDirectory(working_directory):
  67   """Creates a directory 'bisect' as a subdirectory of 'working_directory'.  If
  68   the function is successful, the current working directory will change to that
  69   of the new 'bisect' directory.
  70
  71   Returns:
  72     True if the directory was successfully created (or already existed).
  73   """
  74   cwd = os.getcwd()
  75   os.chdir(working_directory)
  76   try:
  77     os.mkdir('bisect')
  78   except OSError, e:
  79     if e.errno != errno.EEXIST:
  80       return False
  81   os.chdir('bisect')
  82   return True
  83
  84
  85 def SubprocessCall(cmd, cwd=None):
  86   """Runs a subprocess with specified parameters.
  87
  88   Args:
  89     params: A list of parameters to pass to gclient.
  90     cwd: Working directory to run from.
  91
  92   Returns:
  93     The return code of the call.
  94   """
  95   if os.name == 'nt':
  96     # "HOME" isn't normally defined on windows, but is needed
  97     # for git to find the user's .netrc file.
  98     if not os.getenv('HOME'):
  99       os.environ['HOME'] = os.environ['USERPROFILE']
 100   shell = os.name == 'nt'
 101   return subprocess.call(cmd, shell=shell, cwd=cwd)
 102
 103
 104 def RunGClient(params, cwd=None):
 105   """Runs gclient with the specified parameters.
 106
 107   Args:
 108     params: A list of parameters to pass to gclient.
 109     cwd: Working directory to run from.
 110
 111   Returns:
 112     The return code of the call.
 113   """
 114   cmd = ['gclient'] + params
 115
 116   return SubprocessCall(cmd, cwd=cwd)
 117
 118
 119 def RunRepo(params):
 120   """Runs cros repo command with specified parameters.
 121
 122   Args:
 123     params: A list of parameters to pass to gclient.
 124
 125   Returns:
 126     The return code of the call.
 127   """
 128   cmd = ['repo'] + params
 129
 130   return SubprocessCall(cmd)
 131
 132
 133 def RunRepoSyncAtTimestamp(timestamp):
 134   """Syncs all git depots to the timestamp specified using repo forall.
 135
 136   Args:
 137     params: Unix timestamp to sync to.
 138
 139   Returns:
 140     The return code of the call.
 141   """
 142   repo_sync = REPO_SYNC_COMMAND % timestamp
 143   cmd = ['forall', '-c', REPO_SYNC_COMMAND % timestamp]
 144   return RunRepo(cmd)
 145
 146
 147 def RunGClientAndCreateConfig(opts, custom_deps=None, cwd=None):
 148   """Runs gclient and creates a config containing both src and src-internal.
 149
 150   Args:
 151     opts: The options parsed from the command line through parse_args().
 152     custom_deps: A dictionary of additional dependencies to add to .gclient.
 153     cwd: Working directory to run from.
 154
 155   Returns:
 156     The return code of the call.
 157   """
 158   spec = GCLIENT_SPEC_DATA
 159
 160   if custom_deps:
 161     for k, v in custom_deps.iteritems():
 162       spec[0]['custom_deps'][k] = v
 163
 164   # Cannot have newlines in string on windows
 165   spec = 'solutions =' + str(spec)
 166   spec = ''.join([l for l in spec.splitlines()])
 167
 168   if opts.target_platform == 'android':
 169     spec += GCLIENT_SPEC_ANDROID
 170
 171   return_code = RunGClient(
 172       ['config', '--spec=%s' % spec, '--git-deps'], cwd=cwd)
 173   return return_code
 174
 175
 176 def IsDepsFileBlink():
 177   """Reads .DEPS.git and returns whether or not we're using blink.
 178
 179   Returns:
 180     True if blink, false if webkit.
 181   """
 182   locals = {'Var': lambda _: locals["vars"][_],
 183             'From': lambda *args: None}
 184   execfile(FILE_DEPS_GIT, {}, locals)
 185   return 'blink.git' in locals['vars']['webkit_url']
 186
 187
 188 def RemoveThirdPartyWebkitDirectory():
 189   """Removes third_party/WebKit.
 190
 191   Returns:
 192     True on success.
 193   """
 194   try:
 195     path_to_dir = os.path.join(os.getcwd(), 'third_party', 'WebKit')
 196     if os.path.exists(path_to_dir):
 197       shutil.rmtree(path_to_dir)
 198   except OSError, e:
 199     if e.errno != errno.ENOENT:
 200       return False
 201   return True
 202
 203
 204 def RunGClientAndSync(reset, cwd=None):
 205   """Runs gclient and does a normal sync.
 206
 207   Args:
 208     reset: Whether to reset any changes to the depot.
 209     cwd: Working directory to run from.
 210
 211   Returns:
 212     The return code of the call.
 213   """
 214   params = ['sync', '--verbose', '--nohooks']
 215   if reset:
 216     params.extend(['--reset', '--force', '--delete_unversioned_trees'])
 217   return RunGClient(params, cwd=cwd)
 218
 219
 220 def SetupGitDepot(opts, reset):
 221   """Sets up the depot for the bisection. The depot will be located in a
 222   subdirectory called 'bisect'.
 223
 224   Args:
 225     opts: The options parsed from the command line through parse_args().
 226     reset: Whether to reset any changes to the depot.
 227
 228   Returns:
 229     True if gclient successfully created the config file and did a sync, False
 230     otherwise.
 231   """
 232   name = 'Setting up Bisection Depot'
 233
 234   if opts.output_buildbot_annotations:
 235     OutputAnnotationStepStart(name)
 236
 237   passed = False
 238
 239   if not RunGClientAndCreateConfig(opts):
 240     passed_deps_check = True
 241     if os.path.isfile(os.path.join('src', FILE_DEPS_GIT)):
 242       cwd = os.getcwd()
 243       os.chdir('src')
 244       if not IsDepsFileBlink():
 245         passed_deps_check = RemoveThirdPartyWebkitDirectory()
 246       else:
 247         passed_deps_check = True
 248       os.chdir(cwd)
 249
 250     if passed_deps_check and not RunGClientAndSync(reset):
 251       passed = True
 252
 253   if opts.output_buildbot_annotations:
 254     print
 255     OutputAnnotationStepClosed()
 256
 257   return passed
 258
 259
 260 def SetupCrosRepo():
 261   """Sets up cros repo for bisecting chromeos.
 262
 263   Returns:
 264     Returns 0 on success.
 265   """
 266   cwd = os.getcwd()
 267   try:
 268     os.mkdir('cros')
 269   except OSError, e:
 270     if e.errno != errno.EEXIST:
 271       return False
 272   os.chdir('cros')
 273
 274   cmd = ['init', '-u'] + REPO_PARAMS
 275
 276   passed = False
 277
 278   if not RunRepo(cmd):
 279     if not RunRepo(['sync']):
 280       passed = True
 281   os.chdir(cwd)
 282
 283   return passed
 284
 285
 286 def SetupAndroidBuildEnvironment(opts):
 287   """Sets up the android build environment.
 288
 289   Args:
 290     opts: The options parsed from the command line through parse_args().
 291     path_to_file: Path to the bisect script's directory.
 292
 293   Returns:
 294     True if successful.
 295   """
 296   path_to_file = os.path.join('build', 'android', 'envsetup.sh')
 297   proc = subprocess.Popen(['bash', '-c', 'source %s && env' % path_to_file],
 298                            stdout=subprocess.PIPE,
 299                            stderr=subprocess.PIPE,
 300                            cwd='src')
 301   (out, _) = proc.communicate()
 302
 303   for line in out.splitlines():
 304     (k, _, v) = line.partition('=')
 305     os.environ[k] = v
 306   return not proc.returncode
 307
 308
 309 def SetupPlatformBuildEnvironment(opts):
 310   """Performs any platform specific setup.
 311
 312   Args:
 313     opts: The options parsed from the command line through parse_args().
 314     path_to_file: Path to the bisect script's directory.
 315
 316   Returns:
 317     True if successful.
 318   """
 319   if opts.target_platform == 'android':
 320     return SetupAndroidBuildEnvironment(opts)
 321   elif opts.target_platform == 'cros':
 322     return SetupCrosRepo()
 323
 324   return True
 325
 326
 327 def CreateBisectDirectoryAndSetupDepot(opts, reset=False):
 328   """Sets up a subdirectory 'bisect' and then retrieves a copy of the depot
 329   there using gclient.
 330
 331   Args:
 332     opts: The options parsed from the command line through parse_args().
 333     reset: Whether to reset any changes to the depot.
 334
 335   Returns:
 336     Returns 0 on success, otherwise 1.
 337   """
 338   if not CreateAndChangeToSourceDirectory(opts.working_directory):
 339     print 'Error: Could not create bisect directory.'
 340     print
 341     return 1
 342
 343   if not SetupGitDepot(opts, reset):
 344     print 'Error: Failed to grab source.'
 345     print
 346     return 1
 347
 348   return 0