tools/bisect_utils.py

   1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Set of operations/utilities related to checking out the depot, and
   6 outputting annotations on the buildbot waterfall. These are intended to be
   7 used by the bisection scripts."""
   8
   9 import errno
  10 import imp
  11 import os
  12 import shutil
  13 import stat
  14 import subprocess
  15 import sys
  16
  17 DEFAULT_GCLIENT_CUSTOM_DEPS = {
  18     "src/data/page_cycler": "https://chrome-internal.googlesource.com/"
  19                             "chrome/data/page_cycler/.git",
  20     "src/data/dom_perf": "https://chrome-internal.googlesource.com/"
  21                          "chrome/data/dom_perf/.git",
  22     "src/data/mach_ports": "https://chrome-internal.googlesource.com/"
  23                            "chrome/data/mach_ports/.git",
  24     "src/tools/perf/data": "https://chrome-internal.googlesource.com/"
  25                            "chrome/tools/perf/data/.git",
  26     "src/third_party/adobe/flash/binaries/ppapi/linux":
  27         "https://chrome-internal.googlesource.com/"
  28         "chrome/deps/adobe/flash/binaries/ppapi/linux/.git",
  29     "src/third_party/adobe/flash/binaries/ppapi/linux_x64":
  30         "https://chrome-internal.googlesource.com/"
  31         "chrome/deps/adobe/flash/binaries/ppapi/linux_x64/.git",
  32     "src/third_party/adobe/flash/binaries/ppapi/mac":
  33         "https://chrome-internal.googlesource.com/"
  34         "chrome/deps/adobe/flash/binaries/ppapi/mac/.git",
  35     "src/third_party/adobe/flash/binaries/ppapi/mac_64":
  36         "https://chrome-internal.googlesource.com/"
  37         "chrome/deps/adobe/flash/binaries/ppapi/mac_64/.git",
  38     "src/third_party/adobe/flash/binaries/ppapi/win":
  39         "https://chrome-internal.googlesource.com/"
  40         "chrome/deps/adobe/flash/binaries/ppapi/win/.git",
  41     "src/third_party/adobe/flash/binaries/ppapi/win_x64":
  42         "https://chrome-internal.googlesource.com/"
  43         "chrome/deps/adobe/flash/binaries/ppapi/win_x64/.git",}
  44
  45 GCLIENT_SPEC_DATA = [
  46   { "name"        : "src",
  47     "url"         : "https://chromium.googlesource.com/chromium/src.git",
  48     "deps_file"   : ".DEPS.git",
  49     "managed"     : True,
  50     "custom_deps" : {},
  51     "safesync_url": "",
  52   },
  53 ]
  54 GCLIENT_SPEC_ANDROID = "\ntarget_os = ['android']"
  55 GCLIENT_CUSTOM_DEPS_V8 = {"src/v8_bleeding_edge": "git://github.com/v8/v8.git"}
  56 FILE_DEPS_GIT = '.DEPS.git'
  57
  58 REPO_PARAMS = [
  59   'https://chrome-internal.googlesource.com/chromeos/manifest-internal/',
  60   '--repo-url',
  61   'https://git.chromium.org/external/repo.git'
  62 ]
  63
  64 REPO_SYNC_COMMAND = 'git checkout -f $(git rev-list --max-count=1 '\
  65                     '--before=%d remotes/m/master)'
  66
  67 ORIGINAL_ENV = {}
  68
  69 def OutputAnnotationStepStart(name):
  70   """Outputs appropriate annotation to signal the start of a step to
  71   a trybot.
  72
  73   Args:
  74     name: The name of the step.
  75   """
  76   print
  77   print '@@@SEED_STEP %s@@@' % name
  78   print '@@@STEP_CURSOR %s@@@' % name
  79   print '@@@STEP_STARTED@@@'
  80   print
  81   sys.stdout.flush()
  82
  83
  84 def OutputAnnotationStepClosed():
  85   """Outputs appropriate annotation to signal the closing of a step to
  86   a trybot."""
  87   print
  88   print '@@@STEP_CLOSED@@@'
  89   print
  90   sys.stdout.flush()
  91
  92
  93 def OutputAnnotationStepLink(label, url):
  94   """Outputs appropriate annotation to print a link.
  95
  96   Args:
  97     label: The name to print.
  98     url: The url to print.
  99   """
 100   print
 101   print '@@@STEP_LINK@%s@%s@@@' % (label, url)
 102   print
 103   sys.stdout.flush()
 104
 105
 106 def LoadExtraSrc(path_to_file):
 107   """Attempts to load an extra source file. If this is successful, uses the
 108   new module to override some global values, such as gclient spec data.
 109
 110   Returns:
 111     The loaded src module, or None."""
 112   try:
 113     global GCLIENT_SPEC_DATA
 114     global GCLIENT_SPEC_ANDROID
 115     extra_src = imp.load_source('data', path_to_file)
 116     GCLIENT_SPEC_DATA = extra_src.GetGClientSpec()
 117     GCLIENT_SPEC_ANDROID = extra_src.GetGClientSpecExtraParams()
 118     return extra_src
 119   except ImportError, e:
 120     return None
 121
 122
 123 def IsTelemetryCommand(command):
 124   """Attempts to discern whether or not a given command is running telemetry."""
 125   return ('tools/perf/run_' in command or 'tools\\perf\\run_' in command)
 126
 127
 128 def CreateAndChangeToSourceDirectory(working_directory):
 129   """Creates a directory 'bisect' as a subdirectory of 'working_directory'.  If
 130   the function is successful, the current working directory will change to that
 131   of the new 'bisect' directory.
 132
 133   Returns:
 134     True if the directory was successfully created (or already existed).
 135   """
 136   cwd = os.getcwd()
 137   os.chdir(working_directory)
 138   try:
 139     os.mkdir('bisect')
 140   except OSError, e:
 141     if e.errno != errno.EEXIST:
 142       return False
 143   os.chdir('bisect')
 144   return True
 145
 146
 147 def SubprocessCall(cmd, cwd=None):
 148   """Runs a subprocess with specified parameters.
 149
 150   Args:
 151     params: A list of parameters to pass to gclient.
 152     cwd: Working directory to run from.
 153
 154   Returns:
 155     The return code of the call.
 156   """
 157   if os.name == 'nt':
 158     # "HOME" isn't normally defined on windows, but is needed
 159     # for git to find the user's .netrc file.
 160     if not os.getenv('HOME'):
 161       os.environ['HOME'] = os.environ['USERPROFILE']
 162   shell = os.name == 'nt'
 163   return subprocess.call(cmd, shell=shell, cwd=cwd)
 164
 165
 166 def RunGClient(params, cwd=None):
 167   """Runs gclient with the specified parameters.
 168
 169   Args:
 170     params: A list of parameters to pass to gclient.
 171     cwd: Working directory to run from.
 172
 173   Returns:
 174     The return code of the call.
 175   """
 176   cmd = ['gclient'] + params
 177
 178   return SubprocessCall(cmd, cwd=cwd)
 179
 180
 181 def RunRepo(params):
 182   """Runs cros repo command with specified parameters.
 183
 184   Args:
 185     params: A list of parameters to pass to gclient.
 186
 187   Returns:
 188     The return code of the call.
 189   """
 190   cmd = ['repo'] + params
 191
 192   return SubprocessCall(cmd)
 193
 194
 195 def RunRepoSyncAtTimestamp(timestamp):
 196   """Syncs all git depots to the timestamp specified using repo forall.
 197
 198   Args:
 199     params: Unix timestamp to sync to.
 200
 201   Returns:
 202     The return code of the call.
 203   """
 204   repo_sync = REPO_SYNC_COMMAND % timestamp
 205   cmd = ['forall', '-c', REPO_SYNC_COMMAND % timestamp]
 206   return RunRepo(cmd)
 207
 208
 209 def RunGClientAndCreateConfig(opts, custom_deps=None, cwd=None):
 210   """Runs gclient and creates a config containing both src and src-internal.
 211
 212   Args:
 213     opts: The options parsed from the command line through parse_args().
 214     custom_deps: A dictionary of additional dependencies to add to .gclient.
 215     cwd: Working directory to run from.
 216
 217   Returns:
 218     The return code of the call.
 219   """
 220   spec = GCLIENT_SPEC_DATA
 221
 222   if custom_deps:
 223     for k, v in custom_deps.iteritems():
 224       spec[0]['custom_deps'][k] = v
 225
 226   # Cannot have newlines in string on windows
 227   spec = 'solutions =' + str(spec)
 228   spec = ''.join([l for l in spec.splitlines()])
 229
 230   if 'android' in opts.target_platform:
 231     spec += GCLIENT_SPEC_ANDROID
 232
 233   return_code = RunGClient(
 234       ['config', '--spec=%s' % spec, '--git-deps'], cwd=cwd)
 235   return return_code
 236
 237
 238 def IsDepsFileBlink():
 239   """Reads .DEPS.git and returns whether or not we're using blink.
 240
 241   Returns:
 242     True if blink, false if webkit.
 243   """
 244   locals = {'Var': lambda _: locals["vars"][_],
 245             'From': lambda *args: None}
 246   execfile(FILE_DEPS_GIT, {}, locals)
 247   return 'blink.git' in locals['vars']['webkit_url']
 248
 249
 250 def RemoveThirdPartyWebkitDirectory():
 251   """Removes third_party/WebKit.
 252
 253   Returns:
 254     True on success.
 255   """
 256   try:
 257     path_to_dir = os.path.join(os.getcwd(), 'third_party', 'WebKit')
 258     if os.path.exists(path_to_dir):
 259       shutil.rmtree(path_to_dir)
 260   except OSError, e:
 261     if e.errno != errno.ENOENT:
 262       return False
 263   return True
 264
 265
 266 def OnAccessError(func, path, exc_info):
 267   """
 268   Source: http://stackoverflow.com/questions/2656322/python-shutil-rmtree-fails-on-windows-with-access-is-denied
 269
 270   Error handler for ``shutil.rmtree``.
 271
 272   If the error is due to an access error (read only file)
 273   it attempts to add write permission and then retries.
 274
 275   If the error is for another reason it re-raises the error.
 276
 277   Args:
 278     func: The function that raised the error.
 279     path: The path name passed to func.
 280     exc_info: Exception information returned by sys.exc_info().
 281   """
 282   if not os.access(path, os.W_OK):
 283     # Is the error an access error ?
 284     os.chmod(path, stat.S_IWUSR)
 285     func(path)
 286   else:
 287     raise
 288
 289
 290 def RemoveThirdPartyLibjingleDirectory():
 291   """Removes third_party/libjingle. At some point, libjingle was causing issues
 292   syncing when using the git workflow (crbug.com/266324).
 293
 294   Returns:
 295     True on success.
 296   """
 297   path_to_dir = os.path.join(os.getcwd(), 'third_party', 'libjingle')
 298   try:
 299     if os.path.exists(path_to_dir):
 300       shutil.rmtree(path_to_dir, onerror=OnAccessError)
 301   except OSError, e:
 302     print 'Error #%d while running shutil.rmtree(%s): %s' % (
 303         e.errno, path_to_dir, str(e))
 304     if e.errno != errno.ENOENT:
 305       return False
 306   return True
 307
 308
 309 def _CleanupPreviousGitRuns():
 310   """Performs necessary cleanup between runs."""
 311   # If a previous run of git crashed, bot was reset, etc... we
 312   # might end up with leftover index.lock files.
 313   for (path, dir, files) in os.walk(os.getcwd()):
 314     for cur_file in files:
 315       if cur_file.endswith('index.lock'):
 316         path_to_file = os.path.join(path, cur_file)
 317         os.remove(path_to_file)
 318
 319
 320 def RunGClientAndSync(cwd=None):
 321   """Runs gclient and does a normal sync.
 322
 323   Args:
 324     cwd: Working directory to run from.
 325
 326   Returns:
 327     The return code of the call.
 328   """
 329   params = ['sync', '--verbose', '--nohooks', '--reset', '--force']
 330   return RunGClient(params, cwd=cwd)
 331
 332
 333 def SetupGitDepot(opts, custom_deps):
 334   """Sets up the depot for the bisection. The depot will be located in a
 335   subdirectory called 'bisect'.
 336
 337   Args:
 338     opts: The options parsed from the command line through parse_args().
 339     custom_deps: A dictionary of additional dependencies to add to .gclient.
 340
 341   Returns:
 342     True if gclient successfully created the config file and did a sync, False
 343     otherwise.
 344   """
 345   name = 'Setting up Bisection Depot'
 346
 347   if opts.output_buildbot_annotations:
 348     OutputAnnotationStepStart(name)
 349
 350   passed = False
 351
 352   if not RunGClientAndCreateConfig(opts, custom_deps):
 353     passed_deps_check = True
 354     if os.path.isfile(os.path.join('src', FILE_DEPS_GIT)):
 355       cwd = os.getcwd()
 356       os.chdir('src')
 357       if not IsDepsFileBlink():
 358         passed_deps_check = RemoveThirdPartyWebkitDirectory()
 359       else:
 360         passed_deps_check = True
 361       if passed_deps_check:
 362         passed_deps_check = RemoveThirdPartyLibjingleDirectory()
 363       os.chdir(cwd)
 364
 365     if passed_deps_check:
 366       _CleanupPreviousGitRuns()
 367
 368       RunGClient(['revert'])
 369       if not RunGClientAndSync():
 370         passed = True
 371
 372   if opts.output_buildbot_annotations:
 373     print
 374     OutputAnnotationStepClosed()
 375
 376   return passed
 377
 378
 379 def SetupCrosRepo():
 380   """Sets up cros repo for bisecting chromeos.
 381
 382   Returns:
 383     Returns 0 on success.
 384   """
 385   cwd = os.getcwd()
 386   try:
 387     os.mkdir('cros')
 388   except OSError, e:
 389     if e.errno != errno.EEXIST:
 390       return False
 391   os.chdir('cros')
 392
 393   cmd = ['init', '-u'] + REPO_PARAMS
 394
 395   passed = False
 396
 397   if not RunRepo(cmd):
 398     if not RunRepo(['sync']):
 399       passed = True
 400   os.chdir(cwd)
 401
 402   return passed
 403
 404
 405 def CopyAndSaveOriginalEnvironmentVars():
 406   """Makes a copy of the current environment variables."""
 407   # TODO: Waiting on crbug.com/255689, will remove this after.
 408   vars_to_remove = []
 409   for k, v in os.environ.iteritems():
 410     if 'ANDROID' in k:
 411       vars_to_remove.append(k)
 412   vars_to_remove.append('CHROME_SRC')
 413   vars_to_remove.append('CHROMIUM_GYP_FILE')
 414   vars_to_remove.append('GYP_CROSSCOMPILE')
 415   vars_to_remove.append('GYP_DEFINES')
 416   vars_to_remove.append('GYP_GENERATORS')
 417   vars_to_remove.append('GYP_GENERATOR_FLAGS')
 418   vars_to_remove.append('OBJCOPY')
 419   for k in vars_to_remove:
 420     if os.environ.has_key(k):
 421       del os.environ[k]
 422
 423   global ORIGINAL_ENV
 424   ORIGINAL_ENV = os.environ.copy()
 425
 426
 427 def SetupAndroidBuildEnvironment(opts, path_to_src=None):
 428   """Sets up the android build environment.
 429
 430   Args:
 431     opts: The options parsed from the command line through parse_args().
 432     path_to_src: Path to the src checkout.
 433
 434   Returns:
 435     True if successful.
 436   """
 437
 438   # Revert the environment variables back to default before setting them up
 439   # with envsetup.sh.
 440   env_vars = os.environ.copy()
 441   for k, _ in env_vars.iteritems():
 442     del os.environ[k]
 443   for k, v in ORIGINAL_ENV.iteritems():
 444     os.environ[k] = v
 445
 446   path_to_file = os.path.join('build', 'android', 'envsetup.sh')
 447   proc = subprocess.Popen(['bash', '-c', 'source %s && env' % path_to_file],
 448                            stdout=subprocess.PIPE,
 449                            stderr=subprocess.PIPE,
 450                            cwd=path_to_src)
 451   (out, _) = proc.communicate()
 452
 453   for line in out.splitlines():
 454     (k, _, v) = line.partition('=')
 455     os.environ[k] = v
 456
 457   return not proc.returncode
 458
 459
 460 def SetupPlatformBuildEnvironment(opts):
 461   """Performs any platform specific setup.
 462
 463   Args:
 464     opts: The options parsed from the command line through parse_args().
 465
 466   Returns:
 467     True if successful.
 468   """
 469   if 'android' in opts.target_platform:
 470     CopyAndSaveOriginalEnvironmentVars()
 471     return SetupAndroidBuildEnvironment(opts)
 472   elif opts.target_platform == 'cros':
 473     return SetupCrosRepo()
 474
 475   return True
 476
 477
 478 def CheckIfBisectDepotExists(opts):
 479   """Checks if the bisect directory already exists.
 480
 481   Args:
 482     opts: The options parsed from the command line through parse_args().
 483
 484   Returns:
 485     Returns True if it exists.
 486   """
 487   path_to_dir = os.path.join(opts.working_directory, 'bisect', 'src')
 488   return os.path.exists(path_to_dir)
 489
 490
 491 def CreateBisectDirectoryAndSetupDepot(opts, custom_deps):
 492   """Sets up a subdirectory 'bisect' and then retrieves a copy of the depot
 493   there using gclient.
 494
 495   Args:
 496     opts: The options parsed from the command line through parse_args().
 497     custom_deps: A dictionary of additional dependencies to add to .gclient.
 498   """
 499   if not CreateAndChangeToSourceDirectory(opts.working_directory):
 500     raise RuntimeError('Could not create bisect directory.')
 501
 502   if not SetupGitDepot(opts, custom_deps):
 503     raise RuntimeError('Failed to grab source.')