tools/bisect_utils.py

   1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Set of operations/utilities related to checking out the depot, and
   6 outputting annotations on the buildbot waterfall. These are intended to be
   7 used by the bisection scripts."""
   8
   9 import errno
  10 import os
  11 import shutil
  12 import subprocess
  13 import sys
  14
  15 GCLIENT_SPEC_DATA = [
  16   { "name"        : "src",
  17     "url"         : "https://chromium.googlesource.com/chromium/src.git",
  18     "deps_file"   : ".DEPS.git",
  19     "managed"     : True,
  20     "custom_deps" : {
  21       "src/data/page_cycler": "https://chrome-internal.googlesource.com/"
  22                               "chrome/data/page_cycler/.git",
  23       "src/data/dom_perf": "https://chrome-internal.googlesource.com/"
  24                            "chrome/data/dom_perf/.git",
  25       "src/data/mach_ports": "https://chrome-internal.googlesource.com/"
  26                            "chrome/data/mach_ports/.git",
  27       "src/tools/perf/data": "https://chrome-internal.googlesource.com/"
  28                              "chrome/tools/perf/data/.git",
  29       "src/third_party/adobe/flash/binaries/ppapi/linux":
  30           "https://chrome-internal.googlesource.com/"
  31           "chrome/deps/adobe/flash/binaries/ppapi/linux/.git",
  32       "src/third_party/adobe/flash/binaries/ppapi/linux_x64":
  33           "https://chrome-internal.googlesource.com/"
  34           "chrome/deps/adobe/flash/binaries/ppapi/linux_x64/.git",
  35       "src/third_party/adobe/flash/binaries/ppapi/mac":
  36           "https://chrome-internal.googlesource.com/"
  37           "chrome/deps/adobe/flash/binaries/ppapi/mac/.git",
  38       "src/third_party/adobe/flash/binaries/ppapi/mac_64":
  39           "https://chrome-internal.googlesource.com/"
  40           "chrome/deps/adobe/flash/binaries/ppapi/mac_64/.git",
  41       "src/third_party/adobe/flash/binaries/ppapi/win":
  42           "https://chrome-internal.googlesource.com/"
  43           "chrome/deps/adobe/flash/binaries/ppapi/win/.git",
  44       "src/third_party/adobe/flash/binaries/ppapi/win_x64":
  45           "https://chrome-internal.googlesource.com/"
  46           "chrome/deps/adobe/flash/binaries/ppapi/win_x64/.git",
  47     },
  48     "safesync_url": "",
  49   },
  50 ]
  51 GCLIENT_SPEC_ANDROID = "\ntarget_os = ['android']"
  52 GCLIENT_CUSTOM_DEPS_V8 = {"src/v8_bleeding_edge": "git://github.com/v8/v8.git"}
  53 FILE_DEPS_GIT = '.DEPS.git'
  54
  55 REPO_PARAMS = [
  56   'https://chrome-internal.googlesource.com/chromeos/manifest-internal/',
  57   '--repo-url',
  58   'https://git.chromium.org/external/repo.git'
  59 ]
  60
  61 REPO_SYNC_COMMAND = 'git checkout -f $(git rev-list --max-count=1 '\
  62                     '--before=%d remotes/m/master)'
  63
  64 ORIGINAL_ENV = {}
  65
  66 def OutputAnnotationStepStart(name):
  67   """Outputs appropriate annotation to signal the start of a step to
  68   a trybot.
  69
  70   Args:
  71     name: The name of the step.
  72   """
  73   print
  74   print '@@@SEED_STEP %s@@@' % name
  75   print '@@@STEP_CURSOR %s@@@' % name
  76   print '@@@STEP_STARTED@@@'
  77   print
  78   sys.stdout.flush()
  79
  80
  81 def OutputAnnotationStepClosed():
  82   """Outputs appropriate annotation to signal the closing of a step to
  83   a trybot."""
  84   print
  85   print '@@@STEP_CLOSED@@@'
  86   print
  87   sys.stdout.flush()
  88
  89
  90 def CreateAndChangeToSourceDirectory(working_directory):
  91   """Creates a directory 'bisect' as a subdirectory of 'working_directory'.  If
  92   the function is successful, the current working directory will change to that
  93   of the new 'bisect' directory.
  94
  95   Returns:
  96     True if the directory was successfully created (or already existed).
  97   """
  98   cwd = os.getcwd()
  99   os.chdir(working_directory)
 100   try:
 101     os.mkdir('bisect')
 102   except OSError, e:
 103     if e.errno != errno.EEXIST:
 104       return False
 105   os.chdir('bisect')
 106   return True
 107
 108
 109 def SubprocessCall(cmd, cwd=None):
 110   """Runs a subprocess with specified parameters.
 111
 112   Args:
 113     params: A list of parameters to pass to gclient.
 114     cwd: Working directory to run from.
 115
 116   Returns:
 117     The return code of the call.
 118   """
 119   if os.name == 'nt':
 120     # "HOME" isn't normally defined on windows, but is needed
 121     # for git to find the user's .netrc file.
 122     if not os.getenv('HOME'):
 123       os.environ['HOME'] = os.environ['USERPROFILE']
 124   shell = os.name == 'nt'
 125   return subprocess.call(cmd, shell=shell, cwd=cwd)
 126
 127
 128 def RunGClient(params, cwd=None):
 129   """Runs gclient with the specified parameters.
 130
 131   Args:
 132     params: A list of parameters to pass to gclient.
 133     cwd: Working directory to run from.
 134
 135   Returns:
 136     The return code of the call.
 137   """
 138   cmd = ['gclient'] + params
 139
 140   return SubprocessCall(cmd, cwd=cwd)
 141
 142
 143 def RunRepo(params):
 144   """Runs cros repo command with specified parameters.
 145
 146   Args:
 147     params: A list of parameters to pass to gclient.
 148
 149   Returns:
 150     The return code of the call.
 151   """
 152   cmd = ['repo'] + params
 153
 154   return SubprocessCall(cmd)
 155
 156
 157 def RunRepoSyncAtTimestamp(timestamp):
 158   """Syncs all git depots to the timestamp specified using repo forall.
 159
 160   Args:
 161     params: Unix timestamp to sync to.
 162
 163   Returns:
 164     The return code of the call.
 165   """
 166   repo_sync = REPO_SYNC_COMMAND % timestamp
 167   cmd = ['forall', '-c', REPO_SYNC_COMMAND % timestamp]
 168   return RunRepo(cmd)
 169
 170
 171 def RunGClientAndCreateConfig(opts, custom_deps=None, cwd=None):
 172   """Runs gclient and creates a config containing both src and src-internal.
 173
 174   Args:
 175     opts: The options parsed from the command line through parse_args().
 176     custom_deps: A dictionary of additional dependencies to add to .gclient.
 177     cwd: Working directory to run from.
 178
 179   Returns:
 180     The return code of the call.
 181   """
 182   spec = GCLIENT_SPEC_DATA
 183
 184   if custom_deps:
 185     for k, v in custom_deps.iteritems():
 186       spec[0]['custom_deps'][k] = v
 187
 188   # Cannot have newlines in string on windows
 189   spec = 'solutions =' + str(spec)
 190   spec = ''.join([l for l in spec.splitlines()])
 191
 192   if opts.target_platform == 'android':
 193     spec += GCLIENT_SPEC_ANDROID
 194
 195   return_code = RunGClient(
 196       ['config', '--spec=%s' % spec, '--git-deps'], cwd=cwd)
 197   return return_code
 198
 199
 200 def IsDepsFileBlink():
 201   """Reads .DEPS.git and returns whether or not we're using blink.
 202
 203   Returns:
 204     True if blink, false if webkit.
 205   """
 206   locals = {'Var': lambda _: locals["vars"][_],
 207             'From': lambda *args: None}
 208   execfile(FILE_DEPS_GIT, {}, locals)
 209   return 'blink.git' in locals['vars']['webkit_url']
 210
 211
 212 def RemoveThirdPartyWebkitDirectory():
 213   """Removes third_party/WebKit.
 214
 215   Returns:
 216     True on success.
 217   """
 218   try:
 219     path_to_dir = os.path.join(os.getcwd(), 'third_party', 'WebKit')
 220     if os.path.exists(path_to_dir):
 221       shutil.rmtree(path_to_dir)
 222   except OSError, e:
 223     if e.errno != errno.ENOENT:
 224       return False
 225   return True
 226
 227
 228 def RunGClientAndSync(cwd=None):
 229   """Runs gclient and does a normal sync.
 230
 231   Args:
 232     cwd: Working directory to run from.
 233
 234   Returns:
 235     The return code of the call.
 236   """
 237   params = ['sync', '--verbose', '--nohooks', '--reset', '--force']
 238   return RunGClient(params, cwd=cwd)
 239
 240
 241 def SetupGitDepot(opts):
 242   """Sets up the depot for the bisection. The depot will be located in a
 243   subdirectory called 'bisect'.
 244
 245   Args:
 246     opts: The options parsed from the command line through parse_args().
 247
 248   Returns:
 249     True if gclient successfully created the config file and did a sync, False
 250     otherwise.
 251   """
 252   name = 'Setting up Bisection Depot'
 253
 254   if opts.output_buildbot_annotations:
 255     OutputAnnotationStepStart(name)
 256
 257   passed = False
 258
 259   if not RunGClientAndCreateConfig(opts):
 260     passed_deps_check = True
 261     if os.path.isfile(os.path.join('src', FILE_DEPS_GIT)):
 262       cwd = os.getcwd()
 263       os.chdir('src')
 264       if not IsDepsFileBlink():
 265         passed_deps_check = RemoveThirdPartyWebkitDirectory()
 266       else:
 267         passed_deps_check = True
 268       os.chdir(cwd)
 269
 270     if passed_deps_check:
 271       RunGClient(['revert'])
 272       if not RunGClientAndSync():
 273         passed = True
 274
 275   if opts.output_buildbot_annotations:
 276     print
 277     OutputAnnotationStepClosed()
 278
 279   return passed
 280
 281
 282 def SetupCrosRepo():
 283   """Sets up cros repo for bisecting chromeos.
 284
 285   Returns:
 286     Returns 0 on success.
 287   """
 288   cwd = os.getcwd()
 289   try:
 290     os.mkdir('cros')
 291   except OSError, e:
 292     if e.errno != errno.EEXIST:
 293       return False
 294   os.chdir('cros')
 295
 296   cmd = ['init', '-u'] + REPO_PARAMS
 297
 298   passed = False
 299
 300   if not RunRepo(cmd):
 301     if not RunRepo(['sync']):
 302       passed = True
 303   os.chdir(cwd)
 304
 305   return passed
 306
 307
 308 def CopyAndSaveOriginalEnvironmentVars():
 309   """Makes a copy of the current environment variables."""
 310   # TODO: Waiting on crbug.com/255689, will remove this after.
 311   vars_to_remove = []
 312   for k, v in os.environ.iteritems():
 313     if 'ANDROID' in k:
 314       vars_to_remove.append(k)
 315   vars_to_remove.append('CHROME_SRC')
 316   vars_to_remove.append('CHROMIUM_GYP_FILE')
 317   vars_to_remove.append('GYP_CROSSCOMPILE')
 318   vars_to_remove.append('GYP_DEFINES')
 319   vars_to_remove.append('GYP_GENERATORS')
 320   vars_to_remove.append('GYP_GENERATOR_FLAGS')
 321   vars_to_remove.append('OBJCOPY')
 322   for k in vars_to_remove:
 323     if os.environ.has_key(k):
 324       del os.environ[k]
 325
 326   global ORIGINAL_ENV
 327   ORIGINAL_ENV = os.environ.copy()
 328
 329
 330 def SetupAndroidBuildEnvironment(opts):
 331   """Sets up the android build environment.
 332
 333   Args:
 334     opts: The options parsed from the command line through parse_args().
 335     path_to_file: Path to the bisect script's directory.
 336
 337   Returns:
 338     True if successful.
 339   """
 340
 341   # Revert the environment variables back to default before setting them up
 342   # with envsetup.sh.
 343   env_vars = os.environ.copy()
 344   for k, _ in env_vars.iteritems():
 345     del os.environ[k]
 346   for k, v in ORIGINAL_ENV.iteritems():
 347     os.environ[k] = v
 348
 349   path_to_file = os.path.join('build', 'android', 'envsetup.sh')
 350   proc = subprocess.Popen(['bash', '-c', 'source %s && env' % path_to_file],
 351                            stdout=subprocess.PIPE,
 352                            stderr=subprocess.PIPE,
 353                            cwd='src')
 354   (out, _) = proc.communicate()
 355
 356   for line in out.splitlines():
 357     (k, _, v) = line.partition('=')
 358     os.environ[k] = v
 359   return not proc.returncode
 360
 361
 362 def SetupPlatformBuildEnvironment(opts):
 363   """Performs any platform specific setup.
 364
 365   Args:
 366     opts: The options parsed from the command line through parse_args().
 367     path_to_file: Path to the bisect script's directory.
 368
 369   Returns:
 370     True if successful.
 371   """
 372   if opts.target_platform == 'android':
 373     CopyAndSaveOriginalEnvironmentVars()
 374     return SetupAndroidBuildEnvironment(opts)
 375   elif opts.target_platform == 'cros':
 376     return SetupCrosRepo()
 377
 378   return True
 379
 380
 381 def CreateBisectDirectoryAndSetupDepot(opts):
 382   """Sets up a subdirectory 'bisect' and then retrieves a copy of the depot
 383   there using gclient.
 384
 385   Args:
 386     opts: The options parsed from the command line through parse_args().
 387     reset: Whether to reset any changes to the depot.
 388
 389   Returns:
 390     Returns 0 on success, otherwise 1.
 391   """
 392   if not CreateAndChangeToSourceDirectory(opts.working_directory):
 393     print 'Error: Could not create bisect directory.'
 394     print
 395     return 1
 396
 397   if not SetupGitDepot(opts):
 398     print 'Error: Failed to grab source.'
 399     print
 400     return 1
 401
 402   return 0