build/landmines.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """
   7 This script runs every build as the first hook (See DEPS). If it detects that
   8 the build should be clobbered, it will delete the contents of the build
   9 directory.
  10
  11 A landmine is tripped when a builder checks out a different revision, and the
  12 diff between the new landmines and the old ones is non-null. At this point, the
  13 build is clobbered.
  14 """
  15
  16 import difflib
  17 import errno
  18 import gyp_environment
  19 import logging
  20 import optparse
  21 import os
  22 import shutil
  23 import sys
  24 import subprocess
  25 import time
  26
  27 import landmine_utils
  28
  29
  30 SRC_DIR = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
  31
  32
  33 def get_build_dir(build_tool, is_iphone=False):
  34   """
  35   Returns output directory absolute path dependent on build and targets.
  36   Examples:
  37     r'c:\b\build\slave\win\build\src\out'
  38     '/mnt/data/b/build/slave/linux/build/src/out'
  39     '/b/build/slave/ios_rel_device/build/src/xcodebuild'
  40
  41   Keep this function in sync with tools/build/scripts/slave/compile.py
  42   """
  43   ret = None
  44   if build_tool == 'xcode':
  45     ret = os.path.join(SRC_DIR, 'xcodebuild')
  46   elif build_tool in ['make', 'ninja', 'ninja-ios']:  # TODO: Remove ninja-ios.
  47     if ('CHROMIUM_OUT_DIR' not in os.environ and
  48         'output_dir' in landmine_utils.gyp_generator_flags()):
  49       output_dir = landmine_utils.gyp_generator_flags()['output_dir']
  50     else:
  51       output_dir = os.environ.get('CHROMIUM_OUT_DIR', 'out')
  52     ret = os.path.join(SRC_DIR, output_dir)
  53   else:
  54     raise NotImplementedError('Unexpected GYP_GENERATORS (%s)' % build_tool)
  55   return os.path.abspath(ret)
  56
  57
  58 def extract_gn_build_commands(build_ninja_file):
  59   """Extracts from a build.ninja the commands to run GN.
  60
  61   The commands to run GN are the gn rule and build.ninja build step at the
  62   top of the build.ninja file. We want to keep these when deleting GN builds
  63   since we want to preserve the command-line flags to GN.
  64
  65   On error, returns the empty string."""
  66   result = ""
  67   with open(build_ninja_file, 'r') as f:
  68     # Read until the second blank line. The first thing GN writes to the file
  69     # is the "rule gn" and the second is the section for "build build.ninja",
  70     # separated by blank lines.
  71     num_blank_lines = 0
  72     while num_blank_lines < 2:
  73       line = f.readline()
  74       if len(line) == 0:
  75         return ''  # Unexpected EOF.
  76       result += line
  77       if line[0] == '\n':
  78         num_blank_lines = num_blank_lines + 1
  79   return result
  80
  81 def delete_build_dir(build_dir):
  82   # GN writes a build.ninja.d file. Note that not all GN builds have args.gn.
  83   build_ninja_d_file = os.path.join(build_dir, 'build.ninja.d')
  84   if not os.path.exists(build_ninja_d_file):
  85     shutil.rmtree(build_dir)
  86     return
  87
  88   # GN builds aren't automatically regenerated when you sync. To avoid
  89   # messing with the GN workflow, erase everything but the args file, and
  90   # write a dummy build.ninja file that will automatically rerun GN the next
  91   # time Ninja is run.
  92   build_ninja_file = os.path.join(build_dir, 'build.ninja')
  93   build_commands = extract_gn_build_commands(build_ninja_file)
  94
  95   try:
  96     gn_args_file = os.path.join(build_dir, 'args.gn')
  97     with open(gn_args_file, 'r') as f:
  98       args_contents = f.read()
  99   except IOError:
 100     args_contents = ''
 101
 102   shutil.rmtree(build_dir)
 103
 104   # Put back the args file (if any).
 105   os.mkdir(build_dir)
 106   if args_contents != '':
 107     with open(gn_args_file, 'w') as f:
 108       f.write(args_contents)
 109
 110   # Write the build.ninja file sufficiently to regenerate itself.
 111   with open(os.path.join(build_dir, 'build.ninja'), 'w') as f:
 112     if build_commands != '':
 113       f.write(build_commands)
 114     else:
 115       # Couldn't parse the build.ninja file, write a default thing.
 116       f.write('''rule gn
 117 command = gn -q gen //out/%s/
 118 description = Regenerating ninja files
 119
 120 build build.ninja: gn
 121 generator = 1
 122 depfile = build.ninja.d
 123 ''' % (os.path.split(build_dir)[1]))
 124
 125   # Write a .d file for the build which references a nonexistant file. This
 126   # will make Ninja always mark the build as dirty.
 127   with open(build_ninja_d_file, 'w') as f:
 128     f.write('build.ninja: nonexistant_file.gn\n')
 129
 130
 131 def clobber_if_necessary(new_landmines):
 132   """Does the work of setting, planting, and triggering landmines."""
 133   out_dir = get_build_dir(landmine_utils.builder())
 134   landmines_path = os.path.normpath(os.path.join(out_dir, '..', '.landmines'))
 135   try:
 136     os.makedirs(out_dir)
 137   except OSError as e:
 138     if e.errno == errno.EEXIST:
 139       pass
 140
 141   if os.path.exists(landmines_path):
 142     with open(landmines_path, 'r') as f:
 143       old_landmines = f.readlines()
 144     if old_landmines != new_landmines:
 145       old_date = time.ctime(os.stat(landmines_path).st_ctime)
 146       diff = difflib.unified_diff(old_landmines, new_landmines,
 147           fromfile='old_landmines', tofile='new_landmines',
 148           fromfiledate=old_date, tofiledate=time.ctime(), n=0)
 149       sys.stdout.write('Clobbering due to:\n')
 150       sys.stdout.writelines(diff)
 151
 152       # Clobber contents of build directory but not directory itself: some
 153       # checkouts have the build directory mounted.
 154       for f in os.listdir(out_dir):
 155         path = os.path.join(out_dir, f)
 156         if os.path.isfile(path):
 157           os.unlink(path)
 158         elif os.path.isdir(path):
 159           delete_build_dir(path)
 160
 161   # Save current set of landmines for next time.
 162   with open(landmines_path, 'w') as f:
 163     f.writelines(new_landmines)
 164
 165
 166 def process_options():
 167   """Returns a list of landmine emitting scripts."""
 168   parser = optparse.OptionParser()
 169   parser.add_option(
 170       '-s', '--landmine-scripts', action='append',
 171       default=[os.path.join(SRC_DIR, 'build', 'get_landmines.py')],
 172       help='Path to the script which emits landmines to stdout. The target '
 173            'is passed to this script via option -t. Note that an extra '
 174            'script can be specified via an env var EXTRA_LANDMINES_SCRIPT.')
 175   parser.add_option('-v', '--verbose', action='store_true',
 176       default=('LANDMINES_VERBOSE' in os.environ),
 177       help=('Emit some extra debugging information (default off). This option '
 178           'is also enabled by the presence of a LANDMINES_VERBOSE environment '
 179           'variable.'))
 180
 181   options, args = parser.parse_args()
 182
 183   if args:
 184     parser.error('Unknown arguments %s' % args)
 185
 186   logging.basicConfig(
 187       level=logging.DEBUG if options.verbose else logging.ERROR)
 188
 189   extra_script = os.environ.get('EXTRA_LANDMINES_SCRIPT')
 190   if extra_script:
 191     return options.landmine_scripts + [extra_script]
 192   else:
 193     return options.landmine_scripts
 194
 195
 196 def main():
 197   landmine_scripts = process_options()
 198
 199   if landmine_utils.builder() in ('dump_dependency_json', 'eclipse'):
 200     return 0
 201
 202   gyp_environment.SetEnvironment()
 203
 204   landmines = []
 205   for s in landmine_scripts:
 206     proc = subprocess.Popen([sys.executable, s], stdout=subprocess.PIPE)
 207     output, _ = proc.communicate()
 208     landmines.extend([('%s\n' % l.strip()) for l in output.splitlines()])
 209   clobber_if_necessary(landmines)
 210
 211   return 0
 212
 213
 214 if __name__ == '__main__':
 215   sys.exit(main())