tools/mb/mb.py

   1 #!/usr/bin/env python
   2 # Copyright 2015 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """MB - the Meta-Build wrapper around GYP and GN
   7
   8 MB is a wrapper script for GYP and GN that can be used to generate build files
   9 for sets of canned configurations and analyze them.
  10 """
  11
  12 from __future__ import print_function
  13
  14 import argparse
  15 import ast
  16 import errno
  17 import json
  18 import os
  19 import pipes
  20 import pprint
  21 import shlex
  22 import shutil
  23 import sys
  24 import subprocess
  25 import tempfile
  26
  27 def main(args):
  28   mbw = MetaBuildWrapper()
  29   mbw.ParseArgs(args)
  30   return mbw.args.func()
  31
  32
  33 class MetaBuildWrapper(object):
  34   def __init__(self):
  35     p = os.path
  36     d = os.path.dirname
  37     self.chromium_src_dir = p.normpath(d(d(d(p.abspath(__file__)))))
  38     self.default_config = p.join(self.chromium_src_dir, 'tools', 'mb',
  39                                  'mb_config.pyl')
  40     self.platform = sys.platform
  41     self.args = argparse.Namespace()
  42     self.configs = {}
  43     self.masters = {}
  44     self.mixins = {}
  45     self.private_configs = []
  46     self.common_dev_configs = []
  47     self.unsupported_configs = []
  48
  49   def ParseArgs(self, argv):
  50     def AddCommonOptions(subp):
  51       subp.add_argument('-b', '--builder',
  52                         help='builder name to look up config from')
  53       subp.add_argument('-m', '--master',
  54                         help='master name to look up config from')
  55       subp.add_argument('-c', '--config',
  56                         help='configuration to analyze')
  57       subp.add_argument('-f', '--config-file', metavar='PATH',
  58                         default=self.default_config,
  59                         help='path to config file '
  60                             '(default is //tools/mb/mb_config.pyl)')
  61       subp.add_argument('-g', '--goma-dir', default=self.ExpandUser('~/goma'),
  62                         help='path to goma directory (default is %(default)s).')
  63       subp.add_argument('-n', '--dryrun', action='store_true',
  64                         help='Do a dry run (i.e., do nothing, just print '
  65                              'the commands that will run)')
  66       subp.add_argument('-q', '--quiet', action='store_true',
  67                         help='Do not print anything on success, '
  68                              'just return an exit code.')
  69       subp.add_argument('-v', '--verbose', action='count',
  70                         help='verbose logging (may specify multiple times).')
  71
  72     parser = argparse.ArgumentParser(prog='mb')
  73     subps = parser.add_subparsers()
  74
  75     subp = subps.add_parser('analyze',
  76                             help='analyze whether changes to a set of files '
  77                                  'will cause a set of binaries to be rebuilt.')
  78     AddCommonOptions(subp)
  79     subp.add_argument('--swarming-targets-file',
  80                       help='save runtime dependencies for targets listed '
  81                            'in file.')
  82     subp.add_argument('path', nargs=1,
  83                       help='path build was generated into.')
  84     subp.add_argument('input_path', nargs=1,
  85                       help='path to a file containing the input arguments '
  86                            'as a JSON object.')
  87     subp.add_argument('output_path', nargs=1,
  88                       help='path to a file containing the output arguments '
  89                            'as a JSON object.')
  90     subp.set_defaults(func=self.CmdAnalyze)
  91
  92     subp = subps.add_parser('gen',
  93                             help='generate a new set of build files')
  94     AddCommonOptions(subp)
  95     subp.add_argument('--swarming-targets-file',
  96                       help='save runtime dependencies for targets listed '
  97                            'in file.')
  98     subp.add_argument('path', nargs=1,
  99                       help='path to generate build into')
 100     subp.set_defaults(func=self.CmdGen)
 101
 102     subp = subps.add_parser('lookup',
 103                             help='look up the command for a given config or '
 104                                  'builder')
 105     AddCommonOptions(subp)
 106     subp.set_defaults(func=self.CmdLookup)
 107
 108     subp = subps.add_parser('validate',
 109                             help='validate the config file')
 110     subp.add_argument('-f', '--config-file', metavar='PATH',
 111                       default=self.default_config,
 112                       help='path to config file '
 113                           '(default is //tools/mb/mb_config.pyl)')
 114     subp.add_argument('-q', '--quiet', action='store_true',
 115                       help='Do not print anything on success, '
 116                            'just return an exit code.')
 117     subp.set_defaults(func=self.CmdValidate)
 118
 119     subp = subps.add_parser('help',
 120                             help='Get help on a subcommand.')
 121     subp.add_argument(nargs='?', action='store', dest='subcommand',
 122                       help='The command to get help for.')
 123     subp.set_defaults(func=self.CmdHelp)
 124
 125     self.args = parser.parse_args(argv)
 126
 127   def CmdAnalyze(self):
 128     vals = self.GetConfig()
 129     if vals['type'] == 'gn':
 130       return self.RunGNAnalyze(vals)
 131     elif vals['type'] == 'gyp':
 132       return self.RunGYPAnalyze(vals)
 133     else:
 134       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 135
 136   def CmdGen(self):
 137     vals = self.GetConfig()
 138     if vals['type'] == 'gn':
 139       return self.RunGNGen(vals)
 140     if vals['type'] == 'gyp':
 141       return self.RunGYPGen(vals)
 142
 143     raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 144
 145   def CmdLookup(self):
 146     vals = self.GetConfig()
 147     if vals['type'] == 'gn':
 148       cmd = self.GNCmd('gen', '<path>', vals['gn_args'])
 149     elif vals['type'] == 'gyp':
 150       if vals['gyp_crosscompile']:
 151         self.Print('GYP_CROSSCOMPILE=1')
 152       cmd = self.GYPCmd('<path>', vals['gyp_defines'], vals['gyp_config'])
 153     else:
 154       raise MBErr('Unknown meta-build type "%s"' % vals['type'])
 155
 156     self.PrintCmd(cmd)
 157     return 0
 158
 159   def CmdHelp(self):
 160     if self.args.subcommand:
 161       self.ParseArgs([self.args.subcommand, '--help'])
 162     else:
 163       self.ParseArgs(['--help'])
 164
 165   def CmdValidate(self):
 166     errs = []
 167
 168     # Read the file to make sure it parses.
 169     self.ReadConfigFile()
 170
 171     # Figure out the whole list of configs and ensure that no config is
 172     # listed in more than one category.
 173     all_configs = {}
 174     for config in self.common_dev_configs:
 175       all_configs[config] = 'common_dev_configs'
 176     for config in self.private_configs:
 177       if config in all_configs:
 178         errs.append('config "%s" listed in "private_configs" also '
 179                     'listed in "%s"' % (config, all_configs['config']))
 180       else:
 181         all_configs[config] = 'private_configs'
 182     for config in self.unsupported_configs:
 183       if config in all_configs:
 184         errs.append('config "%s" listed in "unsupported_configs" also '
 185                     'listed in "%s"' % (config, all_configs['config']))
 186       else:
 187         all_configs[config] = 'unsupported_configs'
 188
 189     for master in self.masters:
 190       for builder in self.masters[master]:
 191         config = self.masters[master][builder]
 192         if config in all_configs and all_configs[config] not in self.masters:
 193           errs.append('Config "%s" used by a bot is also listed in "%s".' %
 194                       (config, all_configs[config]))
 195         else:
 196           all_configs[config] = master
 197
 198     # Check that every referenced config actually exists.
 199     for config, loc in all_configs.items():
 200       if not config in self.configs:
 201         errs.append('Unknown config "%s" referenced from "%s".' %
 202                     (config, loc))
 203
 204     # Check that every actual config is actually referenced.
 205     for config in self.configs:
 206       if not config in all_configs:
 207         errs.append('Unused config "%s".' % config)
 208
 209     # Figure out the whole list of mixins, and check that every mixin
 210     # listed by a config or another mixin actually exists.
 211     referenced_mixins = set()
 212     for config, mixins in self.configs.items():
 213       for mixin in mixins:
 214         if not mixin in self.mixins:
 215           errs.append('Unknown mixin "%s" referenced by config "%s".' %
 216                       (mixin, config))
 217         referenced_mixins.add(mixin)
 218
 219     for mixin in self.mixins:
 220       for sub_mixin in self.mixins[mixin].get('mixins', []):
 221         if not sub_mixin in self.mixins:
 222           errs.append('Unknown mixin "%s" referenced by mixin "%s".' %
 223                       (sub_mixin, mixin))
 224         referenced_mixins.add(sub_mixin)
 225
 226     # Check that every mixin defined is actually referenced somewhere.
 227     for mixin in self.mixins:
 228       if not mixin in referenced_mixins:
 229         errs.append('Unreferenced mixin "%s".' % mixin)
 230
 231     if errs:
 232       raise MBErr(('mb config file %s has problems:' % self.args.config_file) +
 233                     '\n  ' + '\n  '.join(errs))
 234
 235     if not self.args.quiet:
 236       self.Print('mb config file %s looks ok.' % self.args.config_file)
 237     return 0
 238
 239   def GetConfig(self):
 240     self.ReadConfigFile()
 241     config = self.ConfigFromArgs()
 242     if not config in self.configs:
 243       raise MBErr('Config "%s" not found in %s' %
 244                   (config, self.args.config_file))
 245
 246     return self.FlattenConfig(config)
 247
 248   def ReadConfigFile(self):
 249     if not self.Exists(self.args.config_file):
 250       raise MBErr('config file not found at %s' % self.args.config_file)
 251
 252     try:
 253       contents = ast.literal_eval(self.ReadFile(self.args.config_file))
 254     except SyntaxError as e:
 255       raise MBErr('Failed to parse config file "%s": %s' %
 256                  (self.args.config_file, e))
 257
 258     self.common_dev_configs = contents['common_dev_configs']
 259     self.configs = contents['configs']
 260     self.masters = contents['masters']
 261     self.mixins = contents['mixins']
 262     self.private_configs = contents['private_configs']
 263     self.unsupported_configs = contents['unsupported_configs']
 264
 265   def ConfigFromArgs(self):
 266     if self.args.config:
 267       if self.args.master or self.args.builder:
 268         raise MBErr('Can not specific both -c/--config and -m/--master or '
 269                     '-b/--builder')
 270
 271       return self.args.config
 272
 273     if not self.args.master or not self.args.builder:
 274       raise MBErr('Must specify either -c/--config or '
 275                   '(-m/--master and -b/--builder)')
 276
 277     if not self.args.master in self.masters:
 278       raise MBErr('Master name "%s" not found in "%s"' %
 279                   (self.args.master, self.args.config_file))
 280
 281     if not self.args.builder in self.masters[self.args.master]:
 282       raise MBErr('Builder name "%s"  not found under masters[%s] in "%s"' %
 283                   (self.args.builder, self.args.master, self.args.config_file))
 284
 285     return self.masters[self.args.master][self.args.builder]
 286
 287   def FlattenConfig(self, config):
 288     mixins = self.configs[config]
 289     vals = {
 290       'type': None,
 291       'gn_args': [],
 292       'gyp_config': [],
 293       'gyp_defines': [],
 294       'gyp_crosscompile': False,
 295     }
 296
 297     visited = []
 298     self.FlattenMixins(mixins, vals, visited)
 299     return vals
 300
 301   def FlattenMixins(self, mixins, vals, visited):
 302     for m in mixins:
 303       if m not in self.mixins:
 304         raise MBErr('Unknown mixin "%s"' % m)
 305
 306       # TODO: check for cycles in mixins.
 307
 308       visited.append(m)
 309
 310       mixin_vals = self.mixins[m]
 311       if 'type' in mixin_vals:
 312         vals['type'] = mixin_vals['type']
 313       if 'gn_args' in mixin_vals:
 314         if vals['gn_args']:
 315           vals['gn_args'] += ' ' + mixin_vals['gn_args']
 316         else:
 317           vals['gn_args'] = mixin_vals['gn_args']
 318       if 'gyp_config' in mixin_vals:
 319         vals['gyp_config'] = mixin_vals['gyp_config']
 320       if 'gyp_crosscompile' in mixin_vals:
 321         vals['gyp_crosscompile'] = mixin_vals['gyp_crosscompile']
 322       if 'gyp_defines' in mixin_vals:
 323         if vals['gyp_defines']:
 324           vals['gyp_defines'] += ' ' + mixin_vals['gyp_defines']
 325         else:
 326           vals['gyp_defines'] = mixin_vals['gyp_defines']
 327       if 'mixins' in mixin_vals:
 328         self.FlattenMixins(mixin_vals['mixins'], vals, visited)
 329     return vals
 330
 331   def RunGNGen(self, vals):
 332     path = self.args.path[0]
 333
 334     cmd = self.GNCmd('gen', path, vals['gn_args'])
 335
 336     swarming_targets = []
 337     if self.args.swarming_targets_file:
 338       # We need GN to generate the list of runtime dependencies for
 339       # the compile targets listed (one per line) in the file so
 340       # we can run them via swarming. We use ninja_to_gn.pyl to convert
 341       # the compile targets to the matching GN labels.
 342       contents = self.ReadFile(self.args.swarming_targets_file)
 343       swarming_targets = contents.splitlines()
 344       gn_isolate_map = ast.literal_eval(self.ReadFile(os.path.join(
 345           self.chromium_src_dir, 'testing', 'buildbot', 'gn_isolate_map.pyl')))
 346       gn_labels = []
 347       for target in swarming_targets:
 348         if not target in gn_isolate_map:
 349           raise MBErr('test target "%s"  not found in %s' %
 350                       (target, '//testing/buildbot/gn_isolate_map.pyl'))
 351         gn_labels.append(gn_isolate_map[target]['label'])
 352
 353       gn_runtime_deps_path = self.ToAbsPath(path, 'runtime_deps')
 354
 355       # Since GN hasn't run yet, the build directory may not even exist.
 356       self.MaybeMakeDirectory(self.ToAbsPath(path))
 357
 358       self.WriteFile(gn_runtime_deps_path, '\n'.join(gn_labels) + '\n')
 359       cmd.append('--runtime-deps-list-file=%s' % gn_runtime_deps_path)
 360
 361     ret, _, _ = self.Run(cmd)
 362
 363     for target in swarming_targets:
 364       if gn_isolate_map[target]['type'] == 'gpu_browser_test':
 365         runtime_deps_target = 'browser_tests'
 366       else:
 367         runtime_deps_target = target
 368       if sys.platform == 'win32':
 369         deps_path = self.ToAbsPath(path,
 370                                    runtime_deps_target + '.exe.runtime_deps')
 371       else:
 372         deps_path = self.ToAbsPath(path,
 373                                    runtime_deps_target + '.runtime_deps')
 374       if not self.Exists(deps_path):
 375           raise MBErr('did not generate %s' % deps_path)
 376
 377       command, extra_files = self.GetIsolateCommand(target, vals,
 378                                                     gn_isolate_map)
 379
 380       runtime_deps = self.ReadFile(deps_path).splitlines()
 381
 382       isolate_path = self.ToAbsPath(path, target + '.isolate')
 383       self.WriteFile(isolate_path,
 384         pprint.pformat({
 385           'variables': {
 386             'command': command,
 387             'files': sorted(runtime_deps + extra_files),
 388           }
 389         }) + '\n')
 390
 391       self.WriteJSON(
 392         {
 393           'args': [
 394             '--isolated',
 395             self.ToSrcRelPath('%s%s%s.isolated' % (path, os.sep, target)),
 396             '--isolate',
 397             self.ToSrcRelPath('%s%s%s.isolate' % (path, os.sep, target)),
 398           ],
 399           'dir': self.chromium_src_dir,
 400           'version': 1,
 401         },
 402         isolate_path + 'd.gen.json',
 403       )
 404
 405
 406     return ret
 407
 408   def GNCmd(self, subcommand, path, gn_args=''):
 409     if self.platform == 'linux2':
 410       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'linux64',
 411                              'gn')
 412     elif self.platform == 'darwin':
 413       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'mac',
 414                              'gn')
 415     else:
 416       gn_path = os.path.join(self.chromium_src_dir, 'buildtools', 'win',
 417                              'gn.exe')
 418
 419     cmd = [gn_path, subcommand, path]
 420     gn_args = gn_args.replace("$(goma_dir)", self.args.goma_dir)
 421     if gn_args:
 422       cmd.append('--args=%s' % gn_args)
 423     return cmd
 424
 425   def RunGYPGen(self, vals):
 426     path = self.args.path[0]
 427
 428     output_dir, gyp_config = self.ParseGYPConfigPath(path)
 429     if gyp_config != vals['gyp_config']:
 430       raise MBErr('The last component of the path (%s) must match the '
 431                   'GYP configuration specified in the config (%s), and '
 432                   'it does not.' % (gyp_config, vals['gyp_config']))
 433     cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
 434     env = None
 435     if vals['gyp_crosscompile']:
 436       if self.args.verbose:
 437         self.Print('Setting GYP_CROSSCOMPILE=1 in the environment')
 438       env = os.environ.copy()
 439       env['GYP_CROSSCOMPILE'] = '1'
 440     ret, _, _ = self.Run(cmd, env=env)
 441     return ret
 442
 443   def RunGYPAnalyze(self, vals):
 444     output_dir, gyp_config = self.ParseGYPConfigPath(self.args.path[0])
 445     if gyp_config != vals['gyp_config']:
 446       raise MBErr('The last component of the path (%s) must match the '
 447                   'GYP configuration specified in the config (%s), and '
 448                   'it does not.' % (gyp_config, vals['gyp_config']))
 449     if self.args.verbose:
 450       inp = self.ReadInputJSON(['files', 'targets'])
 451       self.Print()
 452       self.Print('analyze input:')
 453       self.PrintJSON(inp)
 454       self.Print()
 455
 456     cmd = self.GYPCmd(output_dir, vals['gyp_defines'], config=gyp_config)
 457     cmd.extend(['-f', 'analyzer',
 458                 '-G', 'config_path=%s' % self.args.input_path[0],
 459                 '-G', 'analyzer_output_path=%s' % self.args.output_path[0]])
 460     ret, _, _ = self.Run(cmd)
 461     if not ret and self.args.verbose:
 462       outp = json.loads(self.ReadFile(self.args.output_path[0]))
 463       self.Print()
 464       self.Print('analyze output:')
 465       self.PrintJSON(outp)
 466       self.Print()
 467
 468     return ret
 469
 470   def RunGNIsolate(self, vals):
 471     build_path = self.args.path[0]
 472     inp = self.ReadInputJSON(['targets'])
 473     if self.args.verbose:
 474       self.Print()
 475       self.Print('isolate input:')
 476       self.PrintJSON(inp)
 477       self.Print()
 478     output_path = self.args.output_path[0]
 479
 480     for target in inp['targets']:
 481       runtime_deps_path = self.ToAbsPath(build_path, target + '.runtime_deps')
 482
 483       if not self.Exists(runtime_deps_path):
 484         self.WriteFailureAndRaise('"%s" does not exist' % runtime_deps_path,
 485                                   output_path)
 486
 487       command, extra_files = self.GetIsolateCommand(target, vals, None)
 488
 489       runtime_deps = self.ReadFile(runtime_deps_path).splitlines()
 490
 491
 492       isolate_path = self.ToAbsPath(build_path, target + '.isolate')
 493       self.WriteFile(isolate_path,
 494         pprint.pformat({
 495           'variables': {
 496             'command': command,
 497             'files': sorted(runtime_deps + extra_files),
 498           }
 499         }) + '\n')
 500
 501       self.WriteJSON(
 502         {
 503           'args': [
 504             '--isolated',
 505             self.ToSrcRelPath('%s/%s.isolated' % (build_path, target)),
 506             '--isolate',
 507             self.ToSrcRelPath('%s/%s.isolate' % (build_path, target)),
 508           ],
 509           'dir': self.chromium_src_dir,
 510           'version': 1,
 511         },
 512         isolate_path + 'd.gen.json',
 513       )
 514
 515     return 0
 516
 517   def GetIsolateCommand(self, target, vals, gn_isolate_map):
 518     # This needs to mirror the settings in //build/config/ui.gni:
 519     # use_x11 = is_linux && !use_ozone.
 520     # TODO(dpranke): Figure out how to keep this in sync better.
 521     use_x11 = (sys.platform == 'linux2' and
 522                not 'target_os="android"' in vals['gn_args'] and
 523                not 'use_ozone=true' in vals['gn_args'])
 524
 525     asan = 'is_asan=true' in vals['gn_args']
 526     msan = 'is_msan=true' in vals['gn_args']
 527     tsan = 'is_tsan=true' in vals['gn_args']
 528
 529     executable_suffix = '.exe' if sys.platform == 'win32' else ''
 530
 531     test_type = gn_isolate_map[target]['type']
 532     cmdline = []
 533     extra_files = []
 534
 535     if use_x11 and test_type == 'windowed_test_launcher':
 536       extra_files = [
 537           'xdisplaycheck',
 538           '../../testing/test_env.py',
 539           '../../testing/xvfb.py',
 540       ]
 541       cmdline = [
 542         '../../testing/xvfb.py',
 543         '.',
 544         './' + str(target),
 545         '--brave-new-test-launcher',
 546         '--test-launcher-bot-mode',
 547         '--asan=%d' % asan,
 548         '--msan=%d' % msan,
 549         '--tsan=%d' % tsan,
 550       ]
 551     elif test_type in ('windowed_test_launcher', 'console_test_launcher'):
 552       extra_files = [
 553           '../../testing/test_env.py'
 554       ]
 555       cmdline = [
 556           '../../testing/test_env.py',
 557           './' + str(target) + executable_suffix,
 558           '--brave-new-test-launcher',
 559           '--test-launcher-bot-mode',
 560           '--asan=%d' % asan,
 561           '--msan=%d' % msan,
 562           '--tsan=%d' % tsan,
 563       ]
 564     elif test_type == 'gpu_browser_test':
 565       extra_files = [
 566           '../../testing/test_env.py'
 567       ]
 568       gtest_filter = gn_isolate_map[target]['gtest_filter']
 569       cmdline = [
 570           '../../testing/test_env.py',
 571           'browser_tests<(EXECUTABLE_SUFFIX)',
 572           '--test-launcher-bot-mode',
 573           '--enable-gpu',
 574           '--test-launcher-jobs=1',
 575           '--gtest_filter=%s' % gtest_filter,
 576       ]
 577     elif test_type in ('raw'):
 578       extra_files = []
 579       cmdline = [
 580           './' + str(target) + executable_suffix,
 581       ] + gn_isolate_map[target].get('args')
 582
 583     else:
 584       self.WriteFailureAndRaise('No command line for %s found (test type %s).'
 585                                 % (target, test_type), output_path=None)
 586
 587     return cmdline, extra_files
 588
 589   def ToAbsPath(self, build_path, *comps):
 590     return os.path.join(self.chromium_src_dir,
 591                         self.ToSrcRelPath(build_path),
 592                         *comps)
 593
 594   def ToSrcRelPath(self, path):
 595     """Returns a relative path from the top of the repo."""
 596     # TODO: Support normal paths in addition to source-absolute paths.
 597     assert(path.startswith('//'))
 598     return path[2:].replace('/', os.sep)
 599
 600   def ParseGYPConfigPath(self, path):
 601     rpath = self.ToSrcRelPath(path)
 602     output_dir, _, config = rpath.rpartition('/')
 603     self.CheckGYPConfigIsSupported(config, path)
 604     return output_dir, config
 605
 606   def CheckGYPConfigIsSupported(self, config, path):
 607     if config not in ('Debug', 'Release'):
 608       if (sys.platform in ('win32', 'cygwin') and
 609           config not in ('Debug_x64', 'Release_x64')):
 610         raise MBErr('Unknown or unsupported config type "%s" in "%s"' %
 611                     config, path)
 612
 613   def GYPCmd(self, output_dir, gyp_defines, config):
 614     gyp_defines = gyp_defines.replace("$(goma_dir)", self.args.goma_dir)
 615     cmd = [
 616         sys.executable,
 617         os.path.join('build', 'gyp_chromium'),
 618         '-G',
 619         'output_dir=' + output_dir,
 620         '-G',
 621         'config=' + config,
 622     ]
 623     for d in shlex.split(gyp_defines):
 624       cmd += ['-D', d]
 625     return cmd
 626
 627   def RunGNAnalyze(self, vals):
 628     # analyze runs before 'gn gen' now, so we need to run gn gen
 629     # in order to ensure that we have a build directory.
 630     ret = self.RunGNGen(vals)
 631     if ret:
 632       return ret
 633
 634     inp = self.ReadInputJSON(['files', 'targets'])
 635     if self.args.verbose:
 636       self.Print()
 637       self.Print('analyze input:')
 638       self.PrintJSON(inp)
 639       self.Print()
 640
 641     output_path = self.args.output_path[0]
 642
 643     # Bail out early if a GN file was modified, since 'gn refs' won't know
 644     # what to do about it.
 645     if any(f.endswith('.gn') or f.endswith('.gni') for f in inp['files']):
 646       self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
 647       return 0
 648
 649     # Bail out early if 'all' was asked for, since 'gn refs' won't recognize it.
 650     if 'all' in inp['targets']:
 651       self.WriteJSON({'status': 'Found dependency (all)'}, output_path)
 652       return 0
 653
 654     # This shouldn't normally happen, but could due to unusual race conditions,
 655     # like a try job that gets scheduled before a patch lands but runs after
 656     # the patch has landed.
 657     if not inp['files']:
 658       self.Print('Warning: No files modified in patch, bailing out early.')
 659       self.WriteJSON({'targets': [],
 660                       'build_targets': [],
 661                       'status': 'No dependency'}, output_path)
 662       return 0
 663
 664     ret = 0
 665     response_file = self.TempFile()
 666     response_file.write('\n'.join(inp['files']) + '\n')
 667     response_file.close()
 668
 669     matching_targets = []
 670     try:
 671       cmd = self.GNCmd('refs', self.args.path[0]) + [
 672           '@%s' % response_file.name, '--all', '--as=output']
 673       ret, out, _ = self.Run(cmd)
 674       if ret and not 'The input matches no targets' in out:
 675         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
 676                                   output_path)
 677       build_dir = self.ToSrcRelPath(self.args.path[0]) + os.sep
 678       for output in out.splitlines():
 679         build_output = output.replace(build_dir, '')
 680         if build_output in inp['targets']:
 681           matching_targets.append(build_output)
 682
 683       cmd = self.GNCmd('refs', self.args.path[0]) + [
 684           '@%s' % response_file.name, '--all']
 685       ret, out, _ = self.Run(cmd)
 686       if ret and not 'The input matches no targets' in out:
 687         self.WriteFailureAndRaise('gn refs returned %d: %s' % (ret, out),
 688                                   output_path)
 689       for label in out.splitlines():
 690         build_target = label[2:]
 691         # We want to accept 'chrome/android:chrome_shell_apk' and
 692         # just 'chrome_shell_apk'. This may result in too many targets
 693         # getting built, but we can adjust that later if need be.
 694         for input_target in inp['targets']:
 695           if (input_target == build_target or
 696               build_target.endswith(':' + input_target)):
 697             matching_targets.append(input_target)
 698     finally:
 699       self.RemoveFile(response_file.name)
 700
 701     if matching_targets:
 702       # TODO: it could be that a target X might depend on a target Y
 703       # and both would be listed in the input, but we would only need
 704       # to specify target X as a build_target (whereas both X and Y are
 705       # targets). I'm not sure if that optimization is generally worth it.
 706       self.WriteJSON({'targets': sorted(matching_targets),
 707                       'build_targets': sorted(matching_targets),
 708                       'status': 'Found dependency'}, output_path)
 709     else:
 710       self.WriteJSON({'targets': [],
 711                       'build_targets': [],
 712                       'status': 'No dependency'}, output_path)
 713
 714     if not ret and self.args.verbose:
 715       outp = json.loads(self.ReadFile(output_path))
 716       self.Print()
 717       self.Print('analyze output:')
 718       self.PrintJSON(outp)
 719       self.Print()
 720
 721     return 0
 722
 723   def ReadInputJSON(self, required_keys):
 724     path = self.args.input_path[0]
 725     output_path = self.args.output_path[0]
 726     if not self.Exists(path):
 727       self.WriteFailureAndRaise('"%s" does not exist' % path, output_path)
 728
 729     try:
 730       inp = json.loads(self.ReadFile(path))
 731     except Exception as e:
 732       self.WriteFailureAndRaise('Failed to read JSON input from "%s": %s' %
 733                                 (path, e), output_path)
 734
 735     for k in required_keys:
 736       if not k in inp:
 737         self.WriteFailureAndRaise('input file is missing a "%s" key' % k,
 738                                   output_path)
 739
 740     return inp
 741
 742   def WriteFailureAndRaise(self, msg, output_path):
 743     if output_path:
 744       self.WriteJSON({'error': msg}, output_path)
 745     raise MBErr(msg)
 746
 747   def WriteJSON(self, obj, path):
 748     try:
 749       self.WriteFile(path, json.dumps(obj, indent=2, sort_keys=True) + '\n')
 750     except Exception as e:
 751       raise MBErr('Error %s writing to the output path "%s"' %
 752                  (e, path))
 753
 754   def PrintCmd(self, cmd):
 755     if cmd[0] == sys.executable:
 756       cmd = ['python'] + cmd[1:]
 757     self.Print(*[pipes.quote(c) for c in cmd])
 758
 759   def PrintJSON(self, obj):
 760     self.Print(json.dumps(obj, indent=2, sort_keys=True))
 761
 762   def Print(self, *args, **kwargs):
 763     # This function largely exists so it can be overridden for testing.
 764     print(*args, **kwargs)
 765
 766   def Run(self, cmd, env=None):
 767     # This function largely exists so it can be overridden for testing.
 768     if self.args.dryrun or self.args.verbose:
 769       self.PrintCmd(cmd)
 770     if self.args.dryrun:
 771       return 0, '', ''
 772     ret, out, err = self.Call(cmd, env=env)
 773     if self.args.verbose:
 774       if out:
 775         self.Print(out, end='')
 776       if err:
 777         self.Print(err, end='', file=sys.stderr)
 778     return ret, out, err
 779
 780   def Call(self, cmd, env=None):
 781     p = subprocess.Popen(cmd, shell=False, cwd=self.chromium_src_dir,
 782                          stdout=subprocess.PIPE, stderr=subprocess.PIPE,
 783                          env=env)
 784     out, err = p.communicate()
 785     return p.returncode, out, err
 786
 787   def ExpandUser(self, path):
 788     # This function largely exists so it can be overridden for testing.
 789     return os.path.expanduser(path)
 790
 791   def Exists(self, path):
 792     # This function largely exists so it can be overridden for testing.
 793     return os.path.exists(path)
 794
 795   def MaybeMakeDirectory(self, path):
 796     try:
 797       os.makedirs(path)
 798     except OSError, e:
 799       if e.errno != errno.EEXIST:
 800         raise
 801
 802   def ReadFile(self, path):
 803     # This function largely exists so it can be overriden for testing.
 804     with open(path) as fp:
 805       return fp.read()
 806
 807   def RemoveFile(self, path):
 808     # This function largely exists so it can be overriden for testing.
 809     os.remove(path)
 810
 811   def TempFile(self, mode='w'):
 812     # This function largely exists so it can be overriden for testing.
 813     return tempfile.NamedTemporaryFile(mode=mode, delete=False)
 814
 815   def WriteFile(self, path, contents):
 816     # This function largely exists so it can be overriden for testing.
 817     if self.args.dryrun or self.args.verbose:
 818       self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path))
 819     with open(path, 'w') as fp:
 820       return fp.write(contents)
 821
 822
 823 class MBErr(Exception):
 824   pass
 825
 826
 827 if __name__ == '__main__':
 828   try:
 829     sys.exit(main(sys.argv[1:]))
 830   except MBErr as e:
 831     print(e)
 832     sys.exit(1)
 833   except KeyboardInterrupt:
 834     print("interrupted, exiting", stream=sys.stderr)
 835     sys.exit(130)