build/gyp_chromium

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 # This script is wrapper for Chromium that adds some support for how GYP
   8 # is invoked by Chromium beyond what can be done in the gclient hooks.
   9
  10 import glob
  11 import gyp_helper
  12 import os
  13 import pipes
  14 import shlex
  15 import subprocess
  16 import string
  17 import sys
  18
  19 script_dir = os.path.dirname(os.path.realpath(__file__))
  20 chrome_src = os.path.abspath(os.path.join(script_dir, os.pardir))
  21
  22 sys.path.insert(0, os.path.join(chrome_src, 'tools', 'gyp', 'pylib'))
  23 import gyp
  24
  25 # Assume this file is in a one-level-deep subdirectory of the source root.
  26 SRC_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  27
  28 # Add paths so that pymod_do_main(...) can import files.
  29 sys.path.insert(1, os.path.join(chrome_src, 'tools'))
  30 sys.path.insert(1, os.path.join(chrome_src, 'tools', 'generate_shim_headers'))
  31 sys.path.insert(1, os.path.join(chrome_src, 'tools', 'grit'))
  32 sys.path.insert(1, os.path.join(chrome_src, 'chrome', 'tools', 'build'))
  33 sys.path.insert(1, os.path.join(chrome_src, 'native_client', 'build'))
  34 sys.path.insert(1, os.path.join(chrome_src, 'native_client_sdk', 'src',
  35     'build_tools'))
  36 sys.path.insert(1, os.path.join(chrome_src, 'remoting', 'tools', 'build'))
  37 sys.path.insert(1, os.path.join(chrome_src, 'third_party', 'liblouis'))
  38 sys.path.insert(1, os.path.join(chrome_src, 'third_party', 'WebKit',
  39     'Source', 'build', 'scripts'))
  40
  41 import find_depot_tools
  42
  43 # On Windows, Psyco shortens warm runs of build/gyp_chromium by about
  44 # 20 seconds on a z600 machine with 12 GB of RAM, from 90 down to 70
  45 # seconds.  Conversely, memory usage of build/gyp_chromium with Psyco
  46 # maxes out at about 158 MB vs. 132 MB without it.
  47 #
  48 # Psyco uses native libraries, so we need to load a different
  49 # installation depending on which OS we are running under. It has not
  50 # been tested whether using Psyco on our Mac and Linux builds is worth
  51 # it (the GYP running time is a lot shorter, so the JIT startup cost
  52 # may not be worth it).
  53 if sys.platform == 'win32':
  54   try:
  55     sys.path.insert(0, os.path.join(chrome_src, 'third_party', 'psyco_win32'))
  56     import psyco
  57   except:
  58     psyco = None
  59 else:
  60   psyco = None
  61
  62
  63 def GetSupplementalFiles():
  64   """Returns a list of the supplemental files that are included in all GYP
  65   sources."""
  66   return glob.glob(os.path.join(chrome_src, '*', 'supplement.gypi'))
  67
  68
  69 def FormatKeyForGN(key):
  70   """Returns the given GYP key reformatted for GN.
  71
  72   GYP dictionary keys can be almost anything, but in GN they are identifiers
  73   and must follow the same rules. This reformats such keys to be valid GN
  74   identifiers."""
  75   return ''.join([c if c in string.ascii_letters else '_' for c in key])
  76
  77
  78 def EscapeStringForGN(s):
  79   """Converts a string to a GN string literal."""
  80   # Escape $ characters which have special meaning to GN.
  81   return '"' + s.replace('$', '\\$').replace('"', '\\"') + '"'
  82
  83
  84 def ProcessGypDefinesItems(items):
  85   """Converts a list of strings to a list of key-value pairs."""
  86   result = []
  87   for item in items:
  88     tokens = item.split('=', 1)
  89     # Some GYP variables have hyphens, which we don't support.
  90     key = FormatKeyForGN(tokens[0])
  91     if len(tokens) == 2:
  92       result += [(key, tokens[1])]
  93     else:
  94       # No value supplied, treat it as a boolean and set it. Note that we
  95       # use the string '1' here so we have a consistent definition whether
  96       # you do 'foo=1' or 'foo'.
  97       result += [(key, '1')]
  98   return result
  99
 100 def GetGypVarsForGN(supplemental_files):
 101   """Returns a dictionary of all GYP vars that we will be passing to GN."""
 102
 103   # GYP defines from the supplemental.gypi files.
 104   supp_items = []
 105   for supplement in supplemental_files:
 106     with open(supplement, 'r') as f:
 107       try:
 108         file_data = eval(f.read(), {'__builtins__': None}, None)
 109       except SyntaxError, e:
 110         e.filename = os.path.abspath(supplement)
 111         raise
 112       variables = file_data.get('variables', [])
 113       for v in variables:
 114         supp_items += [(FormatKeyForGN(v), str(variables[v]))]
 115
 116   # GYP defines from the environment.
 117   env_items = ProcessGypDefinesItems(
 118       shlex.split(os.environ.get('GYP_DEFINES', '')))
 119
 120   # GYP defines from the command line. We can't use optparse since we want
 121   # to ignore all arguments other than "-D".
 122   cmdline_input_items = []
 123   for i in range(len(sys.argv))[1:]:
 124     if sys.argv[i].startswith('-D'):
 125       if sys.argv[i] == '-D' and i + 1 < len(sys.argv):
 126         cmdline_input_items += [sys.argv[i + 1]]
 127       elif len(sys.argv[i]) > 2:
 128         cmdline_input_items += [sys.argv[i][2:]]
 129   cmdline_items = ProcessGypDefinesItems(cmdline_input_items)
 130
 131   return dict(supp_items + env_items + cmdline_items)
 132
 133 def GetOutputDirectory():
 134   """Returns the output directory that GYP will use."""
 135   # GYP generator flags from the command line. We can't use optparse since we
 136   # want to ignore all arguments other than "-G".
 137   needle = '-Goutput_dir='
 138   cmdline_input_items = []
 139   for item in sys.argv[1:]:
 140     if item.startswith(needle):
 141       return item[len(needle):]
 142
 143   env_items = shlex.split(os.environ.get('GYP_GENERATOR_FLAGS', ''))
 144   needle = 'output_dir='
 145   for item in env_items:
 146     if item.startswith(needle):
 147       return item[len(needle):]
 148
 149   return "out"
 150
 151 def GetArgsStringForGN(supplemental_files):
 152   """Returns the args to pass to GN.
 153   Based on a subset of the GYP variables that have been rewritten a bit."""
 154
 155   # Find the .gyp directory in the user's home directory.
 156   home_dot_gyp = os.environ.get('GYP_CONFIG_DIR', None)
 157   if home_dot_gyp:
 158     home_dot_gyp = os.path.expanduser(home_dot_gyp)
 159   if not home_dot_gyp:
 160     home_vars = ['HOME']
 161     if sys.platform in ('cygwin', 'win32'):
 162       home_vars.append('USERPROFILE')
 163     for home_var in home_vars:
 164       home = os.getenv(home_var)
 165       if home != None:
 166         home_dot_gyp = os.path.join(home, '.gyp')
 167         if not os.path.exists(home_dot_gyp):
 168           home_dot_gyp = None
 169         else:
 170           break
 171
 172   if home_dot_gyp:
 173     include_gypi = os.path.join(home_dot_gyp, "include.gypi")
 174     if os.path.exists(include_gypi):
 175       supplemental_files += [include_gypi]
 176
 177   vars_dict = GetGypVarsForGN(supplemental_files)
 178   gn_args = ''
 179
 180   # Note: These are the additional flags passed to various builds by builders
 181   # on the main waterfall. We'll probably need to add these at some point:
 182   #   mac_strip_release=1         http://crbug.com/330301
 183   #   linux_dump_symbols=0        http://crbug.com/330300
 184   #   host_os=linux  Probably can skip, GN knows the host OS.
 185   #   order_text_section=<path>   http://crbug.com/330299
 186   #   chromium_win_pch=0          http://crbug.com/297678
 187   #   chromium_ios_signing=0      http://crbug.com/330302
 188   #   linux_use_tcmalloc=0        http://crbug.com/330303
 189   #   release_extra_flags=...     http://crbug.com/330305
 190
 191   # These tuples of (key, value, gn_arg_string) use the gn_arg_string for
 192   # gn when the key is set to the given value in the GYP arguments.
 193   remap_cases = [
 194       ('android_webview_build', '1', 'is_android_webview_build=true'),
 195       ('branding', 'Chrome', 'is_chrome_branded=true'),
 196       ('build_for_tool', 'drmemory', 'disable_iterator_debugging=true'),
 197       ('build_for_tool', 'tsan', 'disable_iterator_debugging=true'),
 198       ('buildtype', 'Official', 'is_official_build=true'),
 199       ('component', 'shared_library', 'is_component_build=true'),
 200       ('clang', '1', 'is_clang=true'),
 201       ('clang_use_chrome_plugins', '0', 'clang_use_chrome_plugins=false'),
 202       ('disable_glibcxx_debug', '1', 'disable_iterator_debugging=true'),
 203       ('target_arch', 'ia32', 'cpu_arch="x86"'),
 204       ('target_arch', 'x64', 'cpu_arch="x64" force_win64=true'),
 205       ('target_arch', 'arm', 'cpu_arch="arm"'),
 206       ('target_arch', 'mipsel', 'cpu_arch="mipsel"'),
 207       ('fastbuild', '0', 'symbol_level=2'),
 208       ('fastbuild', '1', 'symbol_level=1'),
 209       ('fastbuild', '2', 'symbol_level=0'),
 210       ('OS', 'ios', 'os="ios"'),
 211       ('OS', 'android', 'os="android"'),
 212       ('chromeos', '1', 'os="chromeos"'),
 213       ('use_aura', '1', 'use_aura=true'),
 214       ('use_goma', '1', 'use_goma=true'),
 215       ('asan', '1', 'is_asan=true'),
 216       ('lsan', '1', 'is_lsan=true'),
 217       ('msan', '1', 'is_msan=true'),
 218       ('tsan', '1', 'is_tsan=true'),
 219   ]
 220   for i in remap_cases:
 221     if i[0] in vars_dict and vars_dict[i[0]] == i[1]:
 222       gn_args += ' ' + i[2]
 223
 224   # These string arguments get passed directly as GN strings.
 225   for v in ['android_src', 'windows_sdk_path', 'arm_float_abi']:
 226     if v in vars_dict:
 227       gn_args += ' ' + v + '=' + EscapeStringForGN(vars_dict[v])
 228
 229   # gomadir is renamed goma_dir in the GN build.
 230   if 'gomadir' in vars_dict:
 231     gn_args += ' goma_dir=%s' % EscapeStringForGN(vars_dict['gomadir'])
 232
 233   # These arguments get passed directly as integers (avoiding the quoting and
 234   # escaping of the string ones above).
 235   for v in ['arm_version']:
 236     if v in vars_dict:
 237       gn_args += ' %s=%s' % (v, vars_dict[v])
 238
 239   # Some other flags come from GYP environment variables.
 240   gyp_msvs_version = os.environ.get('GYP_MSVS_VERSION', '')
 241   if gyp_msvs_version:
 242     gn_args += ' visual_studio_version=' + EscapeStringForGN(gyp_msvs_version)
 243   gyp_msvs_override_path = os.environ.get('GYP_MSVS_OVERRIDE_PATH', '')
 244   if gyp_msvs_override_path:
 245     gn_args += ' visual_studio_path=' + \
 246         EscapeStringForGN(gyp_msvs_override_path)
 247
 248   # Set the GYP flag so BUILD files know they're being invoked in GYP mode.
 249   gn_args += ' is_gyp=true'
 250
 251   gyp_outdir = GetOutputDirectory()
 252   gn_args += ' gyp_output_dir=\"%s\"' % gyp_outdir
 253
 254   return gn_args.strip()
 255
 256
 257 def additional_include_files(supplemental_files, args=[]):
 258   """
 259   Returns a list of additional (.gypi) files to include, without duplicating
 260   ones that are already specified on the command line. The list of supplemental
 261   include files is passed in as an argument.
 262   """
 263   # Determine the include files specified on the command line.
 264   # This doesn't cover all the different option formats you can use,
 265   # but it's mainly intended to avoid duplicating flags on the automatic
 266   # makefile regeneration which only uses this format.
 267   specified_includes = set()
 268   for arg in args:
 269     if arg.startswith('-I') and len(arg) > 2:
 270       specified_includes.add(os.path.realpath(arg[2:]))
 271
 272   result = []
 273   def AddInclude(path):
 274     if os.path.realpath(path) not in specified_includes:
 275       result.append(path)
 276
 277   # Always include common.gypi.
 278   AddInclude(os.path.join(script_dir, 'common.gypi'))
 279
 280   # Optionally add supplemental .gypi files if present.
 281   for supplement in supplemental_files:
 282     AddInclude(supplement)
 283
 284   return result
 285
 286
 287 def RunGN(supplemental_includes):
 288   """Runs GN, returning True if it succeeded, printing an error and returning
 289   false if not."""
 290
 291   # The binaries in platform-specific subdirectories in src/tools/gn/bin.
 292   gnpath = SRC_DIR + '/tools/gn/bin/'
 293   if sys.platform in ('cygwin', 'win32'):
 294     gnpath += 'win/gn.exe'
 295   elif sys.platform.startswith('linux'):
 296     # On Linux we have 32-bit and 64-bit versions.
 297     if subprocess.check_output(["getconf", "LONG_BIT"]).find("64") >= 0:
 298       gnpath += 'linux/gn'
 299     else:
 300       gnpath += 'linux/gn32'
 301   elif sys.platform == 'darwin':
 302     gnpath += 'mac/gn'
 303   else:
 304     print 'Unknown platform for GN: ', sys.platform
 305     return False
 306
 307   print 'Generating gyp files from GN...'
 308
 309   # Need to pass both the source root (the bots don't run this command from
 310   # within the source tree) as well as set the is_gyp value so the BUILD files
 311   # to know they're being run under GYP.
 312   args = [gnpath, 'gyp', '-q',
 313           '--root=' + chrome_src,
 314           '--args=' + GetArgsStringForGN(supplemental_includes),
 315           '--output=//' + GetOutputDirectory() + '/gn_build/']
 316   return subprocess.call(args) == 0
 317
 318
 319 if __name__ == '__main__':
 320   args = sys.argv[1:]
 321
 322   if int(os.environ.get('GYP_CHROMIUM_NO_ACTION', 0)):
 323     print 'Skipping gyp_chromium due to GYP_CHROMIUM_NO_ACTION env var.'
 324     sys.exit(0)
 325
 326   # Use the Psyco JIT if available.
 327   if psyco:
 328     psyco.profile()
 329     print "Enabled Psyco JIT."
 330
 331   # Fall back on hermetic python if we happen to get run under cygwin.
 332   # TODO(bradnelson): take this out once this issue is fixed:
 333   #    http://code.google.com/p/gyp/issues/detail?id=177
 334   if sys.platform == 'cygwin':
 335     depot_tools_path = find_depot_tools.add_depot_tools_to_path()
 336     python_dir = sorted(glob.glob(os.path.join(depot_tools_path,
 337                                                'python2*_bin')))[-1]
 338     env = os.environ.copy()
 339     env['PATH'] = python_dir + os.pathsep + env.get('PATH', '')
 340     p = subprocess.Popen(
 341        [os.path.join(python_dir, 'python.exe')] + sys.argv,
 342        env=env, shell=False)
 343     p.communicate()
 344     sys.exit(p.returncode)
 345
 346   gyp_helper.apply_chromium_gyp_env()
 347
 348   # This could give false positives since it doesn't actually do real option
 349   # parsing.  Oh well.
 350   gyp_file_specified = False
 351   for arg in args:
 352     if arg.endswith('.gyp'):
 353       gyp_file_specified = True
 354       break
 355
 356   # If we didn't get a file, check an env var, and then fall back to
 357   # assuming 'all.gyp' from the same directory as the script.
 358   if not gyp_file_specified:
 359     gyp_file = os.environ.get('CHROMIUM_GYP_FILE')
 360     if gyp_file:
 361       # Note that CHROMIUM_GYP_FILE values can't have backslashes as
 362       # path separators even on Windows due to the use of shlex.split().
 363       args.extend(shlex.split(gyp_file))
 364     else:
 365       args.append(os.path.join(script_dir, 'all.gyp'))
 366
 367   # There shouldn't be a circular dependency relationship between .gyp files,
 368   # but in Chromium's .gyp files, on non-Mac platforms, circular relationships
 369   # currently exist.  The check for circular dependencies is currently
 370   # bypassed on other platforms, but is left enabled on the Mac, where a
 371   # violation of the rule causes Xcode to misbehave badly.
 372   # TODO(mark): Find and kill remaining circular dependencies, and remove this
 373   # option.  http://crbug.com/35878.
 374   # TODO(tc): Fix circular dependencies in ChromiumOS then add linux2 to the
 375   # list.
 376   if sys.platform not in ('darwin',):
 377     args.append('--no-circular-check')
 378
 379   # Default to ninja on linux and windows, but only if no generator has
 380   # explicitly been set.
 381   # Also default to ninja on mac, but only when not building chrome/ios.
 382   # . -f / --format has precedence over the env var, no need to check for it
 383   # . set the env var only if it hasn't been set yet
 384   # . chromium.gyp_env has been applied to os.environ at this point already
 385   if sys.platform.startswith('linux') and not os.environ.get('GYP_GENERATORS'):
 386     os.environ['GYP_GENERATORS'] = 'ninja'
 387   if sys.platform.startswith('win') and not os.environ.get('GYP_GENERATORS'):
 388     os.environ['GYP_GENERATORS'] = 'ninja'
 389   elif sys.platform == 'darwin' and not os.environ.get('GYP_GENERATORS') and \
 390       not 'OS=ios' in os.environ.get('GYP_DEFINES', []):
 391     os.environ['GYP_GENERATORS'] = 'ninja'
 392
 393   # If using ninja on windows, and the automatic toolchain has been installed
 394   # by depot_tools, then use it.
 395   if (sys.platform in ('win32', 'cygwin') and
 396       os.environ.get('GYP_GENERATORS') == 'ninja'):
 397     depot_tools_path = find_depot_tools.add_depot_tools_to_path()
 398     toolchain = os.path.normpath(os.path.join(
 399         depot_tools_path, 'win_toolchain', 'vs2013_files'))
 400     if os.path.isdir(toolchain):
 401       os.environ['GYP_MSVS_OVERRIDE_PATH'] = toolchain
 402       os.environ['GYP_MSVS_VERSION'] = '2013'
 403       # We need to make sure windows_sdk_path is set to the automated
 404       # toolchain values in GYP_DEFINES, but don't want to override any other
 405       # values there.
 406       gyp_defines_dict = gyp.NameValueListToDict(gyp.ShlexEnv('GYP_DEFINES'))
 407       win8sdk = os.path.join(toolchain, 'win8sdk')
 408       gyp_defines_dict['windows_sdk_path'] = win8sdk
 409       os.environ['WINDOWSSDKDIR'] = win8sdk
 410       os.environ['GYP_DEFINES'] = ' '.join('%s=%s' % (k, pipes.quote(str(v)))
 411           for k, v in gyp_defines_dict.iteritems())
 412       # Include the VS runtime in the PATH in case it's not machine-installed.
 413       runtime_path = ';'.join(os.path.normpath(os.path.join(toolchain, s))
 414                               for s in ('sys64', 'sys32'))
 415       os.environ['PATH'] = runtime_path + ';' + os.environ['PATH']
 416       print('Using automatic toolchain in %s.' % toolchain)
 417
 418   # If CHROMIUM_GYP_SYNTAX_CHECK is set to 1, it will invoke gyp with --check
 419   # to enfore syntax checking.
 420   syntax_check = os.environ.get('CHROMIUM_GYP_SYNTAX_CHECK')
 421   if syntax_check and int(syntax_check):
 422     args.append('--check')
 423
 424   supplemental_includes = GetSupplementalFiles()
 425   if not RunGN(supplemental_includes):
 426     sys.exit(1)
 427   args.extend(
 428       ['-I' + i for i in additional_include_files(supplemental_includes, args)])
 429
 430   args.extend(['-D', 'gyp_output_dir=' + GetOutputDirectory()])
 431
 432   print 'Updating projects from gyp files...'
 433   sys.stdout.flush()
 434
 435   # Off we go...
 436   sys.exit(gyp.main(args))