tools/isolate_driver.py

   1 #!/usr/bin/env python
   2 # Copyright 2014 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Adaptor script called through build/isolate.gypi.
   7
   8 Creates a wrapping .isolate which 'includes' the original one, that can be
   9 consumed by tools/swarming_client/isolate.py. Path variables are determined
  10 based on the current working directory. The relative_cwd in the .isolated file
  11 is determined based on the .isolate file that declare the 'command' variable to
  12 be used so the wrapping .isolate doesn't affect this value.
  13
  14 This script loads build.ninja and processes it to determine all the executables
  15 referenced by the isolated target. It adds them in the wrapping .isolate file.
  16 """
  17
  18 import StringIO
  19 import glob
  20 import logging
  21 import os
  22 import posixpath
  23 import subprocess
  24 import sys
  25 import time
  26
  27 TOOLS_DIR = os.path.dirname(os.path.abspath(__file__))
  28 SWARMING_CLIENT_DIR = os.path.join(TOOLS_DIR, 'swarming_client')
  29 SRC_DIR = os.path.dirname(TOOLS_DIR)
  30
  31 sys.path.insert(0, SWARMING_CLIENT_DIR)
  32
  33 import isolate_format
  34
  35
  36 def load_ninja_recursively(build_dir, ninja_path, build_steps):
  37   """Crudely extracts all the subninja and build referenced in ninja_path.
  38
  39   In particular, it ignores rule and variable declarations. The goal is to be
  40   performant (well, as much as python can be performant) which is currently in
  41   the <200ms range for a complete chromium tree. As such the code is laid out
  42   for performance instead of readability.
  43   """
  44   logging.debug('Loading %s', ninja_path)
  45   try:
  46     with open(os.path.join(build_dir, ninja_path), 'rb') as f:
  47       line = None
  48       merge_line = ''
  49       subninja = []
  50       for line in f:
  51         line = line.rstrip()
  52         if not line:
  53           continue
  54
  55         if line[-1] == '$':
  56           # The next line needs to be merged in.
  57           merge_line += line[:-1]
  58           continue
  59
  60         if merge_line:
  61           line = merge_line + line
  62           merge_line = ''
  63
  64         statement = line[:line.find(' ')]
  65         if statement == 'build':
  66           # Save the dependency list as a raw string. Only the lines needed will
  67           # be processed with raw_build_to_deps(). This saves a good 70ms of
  68           # processing time.
  69           build_target, dependencies = line[6:].split(': ', 1)
  70           # Interestingly, trying to be smart and only saving the build steps
  71           # with the intended extensions ('', '.stamp', '.so') slows down
  72           # parsing even if 90% of the build rules can be skipped.
  73           # On Windows, a single step may generate two target, so split items
  74           # accordingly. It has only been seen for .exe/.exe.pdb combos.
  75           for i in build_target.strip().split():
  76             build_steps[i] = dependencies
  77         elif statement == 'subninja':
  78           subninja.append(line[9:])
  79   except IOError:
  80     print >> sys.stderr, 'Failed to open %s' % ninja_path
  81     raise
  82
  83   total = 1
  84   for rel_path in subninja:
  85     try:
  86       # Load each of the files referenced.
  87       # TODO(maruel): Skip the files known to not be needed. It saves an aweful
  88       # lot of processing time.
  89       total += load_ninja_recursively(build_dir, rel_path, build_steps)
  90     except IOError:
  91       print >> sys.stderr, '... as referenced by %s' % ninja_path
  92       raise
  93   return total
  94
  95
  96 def load_ninja(build_dir):
  97   """Loads the tree of .ninja files in build_dir."""
  98   build_steps = {}
  99   total = load_ninja_recursively(build_dir, 'build.ninja', build_steps)
 100   logging.info('Loaded %d ninja files, %d build steps', total, len(build_steps))
 101   return build_steps
 102
 103
 104 def using_blacklist(item):
 105   """Returns True if an item should be analyzed.
 106
 107   Ignores many rules that are assumed to not depend on a dynamic library. If
 108   the assumption doesn't hold true anymore for a file format, remove it from
 109   this list. This is simply an optimization.
 110   """
 111   IGNORED = (
 112     '.a', '.cc', '.css', '.def', '.h', '.html', '.js', '.json', '.manifest',
 113     '.o', '.obj', '.pak', '.png', '.pdb', '.strings', '.txt',
 114   )
 115   # ninja files use native path format.
 116   ext = os.path.splitext(item)[1]
 117   if ext in IGNORED:
 118     return False
 119   # Special case Windows, keep .dll.lib but discard .lib.
 120   if item.endswith('.dll.lib'):
 121     return True
 122   if ext == '.lib':
 123     return False
 124   return item not in ('', '|', '||')
 125
 126
 127 def raw_build_to_deps(item):
 128   """Converts a raw ninja build statement into the list of interesting
 129   dependencies.
 130   """
 131   # TODO(maruel): Use a whitelist instead? .stamp, .so.TOC, .dylib.TOC,
 132   # .dll.lib, .exe and empty.
 133   # The first item is the build rule, e.g. 'link', 'cxx', 'phony', etc.
 134   return filter(using_blacklist, item.split(' ')[1:])
 135
 136
 137 def recurse(target, build_steps, rules_seen):
 138   """Recursively returns all the interesting dependencies for root_item."""
 139   out = []
 140   if rules_seen is None:
 141     rules_seen = set()
 142   if target in rules_seen:
 143     # TODO(maruel): Figure out how it happens.
 144     logging.warning('Circular dependency for %s!', target)
 145     return []
 146   rules_seen.add(target)
 147   try:
 148     dependencies = raw_build_to_deps(build_steps[target])
 149   except KeyError:
 150     logging.info('Failed to find a build step to generate: %s', target)
 151     return []
 152   logging.debug('recurse(%s) -> %s', target, dependencies)
 153   for dependency in dependencies:
 154     out.append(dependency)
 155     dependency_raw_dependencies = build_steps.get(dependency)
 156     if dependency_raw_dependencies:
 157       for i in raw_build_to_deps(dependency_raw_dependencies):
 158         out.extend(recurse(i, build_steps, rules_seen))
 159     else:
 160       logging.info('Failed to find a build step to generate: %s', dependency)
 161   return out
 162
 163
 164 def post_process_deps(build_dir, dependencies):
 165   """Processes the dependency list with OS specific rules."""
 166   def filter_item(i):
 167     if i.endswith('.so.TOC'):
 168       # Remove only the suffix .TOC, not the .so!
 169       return i[:-4]
 170     if i.endswith('.dylib.TOC'):
 171       # Remove only the suffix .TOC, not the .dylib!
 172       return i[:-4]
 173     if i.endswith('.dll.lib'):
 174       # Remove only the suffix .lib, not the .dll!
 175       return i[:-4]
 176     return i
 177
 178   # Check for execute access. This gets rid of all the phony rules.
 179   return [
 180     i for i in map(filter_item, dependencies)
 181     if os.access(os.path.join(build_dir, i), os.X_OK)
 182   ]
 183
 184
 185 def create_wrapper(args, isolate_index, isolated_index):
 186   """Creates a wrapper .isolate that add dynamic libs.
 187
 188   The original .isolate is not modified.
 189   """
 190   cwd = os.getcwd()
 191   isolate = args[isolate_index]
 192   # The code assumes the .isolate file is always specified path-less in cwd. Fix
 193   # if this assumption doesn't hold true.
 194   assert os.path.basename(isolate) == isolate, isolate
 195
 196   # This will look like ../out/Debug. This is based against cwd. Note that this
 197   # must equal the value provided as PRODUCT_DIR.
 198   build_dir = os.path.dirname(args[isolated_index])
 199
 200   # This will look like chrome/unit_tests.isolate. It is based against SRC_DIR.
 201   # It's used to calculate temp_isolate.
 202   src_isolate = os.path.relpath(os.path.join(cwd, isolate), SRC_DIR)
 203
 204   # The wrapping .isolate. This will look like
 205   # ../out/Debug/gen/chrome/unit_tests.isolate.
 206   temp_isolate = os.path.join(build_dir, 'gen', src_isolate)
 207   temp_isolate_dir = os.path.dirname(temp_isolate)
 208
 209   # Relative path between the new and old .isolate file.
 210   isolate_relpath = os.path.relpath(
 211       '.', temp_isolate_dir).replace(os.path.sep, '/')
 212
 213   # It's a big assumption here that the name of the isolate file matches the
 214   # primary target. Fix accordingly if this doesn't hold true.
 215   target = isolate[:-len('.isolate')]
 216   build_steps = load_ninja(build_dir)
 217   binary_deps = post_process_deps(build_dir, recurse(target, build_steps, None))
 218   logging.debug(
 219       'Binary dependencies:%s', ''.join('\n  ' + i for i in binary_deps))
 220
 221   # Now do actual wrapping .isolate.
 222   isolate_dict = {
 223     'includes': [
 224       posixpath.join(isolate_relpath, isolate),
 225     ],
 226     'variables': {
 227       # Will look like ['<(PRODUCT_DIR)/lib/flibuser_prefs.so'].
 228       isolate_format.KEY_TRACKED: sorted(
 229           '<(PRODUCT_DIR)/%s' % i.replace(os.path.sep, '/')
 230           for i in binary_deps),
 231     },
 232   }
 233   if not os.path.isdir(temp_isolate_dir):
 234     os.makedirs(temp_isolate_dir)
 235   comment = (
 236       '# Warning: this file was AUTOGENERATED.\n'
 237       '# DO NO EDIT.\n')
 238   out = StringIO.StringIO()
 239   isolate_format.print_all(comment, isolate_dict, out)
 240   isolate_content = out.getvalue()
 241   with open(temp_isolate, 'wb') as f:
 242     f.write(isolate_content)
 243   logging.info('Added %d dynamic libs', len(binary_deps))
 244   logging.debug('%s', isolate_content)
 245   args[isolate_index] = temp_isolate
 246
 247
 248 def main():
 249   logging.basicConfig(level=logging.ERROR, format='%(levelname)7s %(message)s')
 250   args = sys.argv[1:]
 251   isolate = None
 252   isolated = None
 253   is_component = False
 254   for i, arg in enumerate(args):
 255     if arg == '--isolate':
 256       isolate = i + 1
 257     if arg == '--isolated':
 258       isolated = i + 1
 259     if arg == 'component=shared_library':
 260       is_component = True
 261   if isolate is None or isolated is None:
 262     print >> sys.stderr, 'Internal failure'
 263     return 1
 264
 265   if is_component:
 266     create_wrapper(args, isolate, isolated)
 267
 268   swarming_client = os.path.join(SRC_DIR, 'tools', 'swarming_client')
 269   sys.stdout.flush()
 270   result = subprocess.call(
 271       [sys.executable, os.path.join(swarming_client, 'isolate.py')] + args)
 272   return result
 273
 274
 275 if __name__ == '__main__':
 276   sys.exit(main())