build/gyp_chromium

   1 #!/usr/bin/env python
   2
   3 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   4 # Use of this source code is governed by a BSD-style license that can be
   5 # found in the LICENSE file.
   6
   7 # This script is wrapper for Chromium that adds some support for how GYP
   8 # is invoked by Chromium beyond what can be done in the gclient hooks.
   9
  10 import glob
  11 import gyp_helper
  12 import os
  13 import re
  14 import shlex
  15 import subprocess
  16 import string
  17 import sys
  18 import vs_toolchain
  19
  20 script_dir = os.path.dirname(os.path.realpath(__file__))
  21 chrome_src = os.path.abspath(os.path.join(script_dir, os.pardir))
  22
  23 sys.path.insert(0, os.path.join(chrome_src, 'tools', 'gyp', 'pylib'))
  24 import gyp
  25
  26 # Assume this file is in a one-level-deep subdirectory of the source root.
  27 SRC_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  28
  29 # Add paths so that pymod_do_main(...) can import files.
  30 sys.path.insert(1, os.path.join(chrome_src, 'tools'))
  31 sys.path.insert(1, os.path.join(chrome_src, 'tools', 'generate_shim_headers'))
  32 sys.path.insert(1, os.path.join(chrome_src, 'tools', 'grit'))
  33 sys.path.insert(1, os.path.join(chrome_src, 'chrome', 'tools', 'build'))
  34 sys.path.insert(1, os.path.join(chrome_src, 'native_client', 'build'))
  35 sys.path.insert(1, os.path.join(chrome_src, 'native_client_sdk', 'src',
  36     'build_tools'))
  37 sys.path.insert(1, os.path.join(chrome_src, 'remoting', 'tools', 'build'))
  38 sys.path.insert(1, os.path.join(chrome_src, 'third_party', 'liblouis'))
  39 sys.path.insert(1, os.path.join(chrome_src, 'third_party', 'WebKit',
  40     'Source', 'build', 'scripts'))
  41
  42 # On Windows, Psyco shortens warm runs of build/gyp_chromium by about
  43 # 20 seconds on a z600 machine with 12 GB of RAM, from 90 down to 70
  44 # seconds.  Conversely, memory usage of build/gyp_chromium with Psyco
  45 # maxes out at about 158 MB vs. 132 MB without it.
  46 #
  47 # Psyco uses native libraries, so we need to load a different
  48 # installation depending on which OS we are running under. It has not
  49 # been tested whether using Psyco on our Mac and Linux builds is worth
  50 # it (the GYP running time is a lot shorter, so the JIT startup cost
  51 # may not be worth it).
  52 if sys.platform == 'win32':
  53   try:
  54     sys.path.insert(0, os.path.join(chrome_src, 'third_party', 'psyco_win32'))
  55     import psyco
  56   except:
  57     psyco = None
  58 else:
  59   psyco = None
  60
  61
  62 def GetSupplementalFiles():
  63   """Returns a list of the supplemental files that are included in all GYP
  64   sources."""
  65   return glob.glob(os.path.join(chrome_src, '*', 'supplement.gypi'))
  66
  67
  68 def ProcessGypDefinesItems(items):
  69   """Converts a list of strings to a list of key-value pairs."""
  70   result = []
  71   for item in items:
  72     tokens = item.split('=', 1)
  73     # Some GYP variables have hyphens, which we don't support.
  74     if len(tokens) == 2:
  75       result += [(tokens[0], tokens[1])]
  76     else:
  77       # No value supplied, treat it as a boolean and set it. Note that we
  78       # use the string '1' here so we have a consistent definition whether
  79       # you do 'foo=1' or 'foo'.
  80       result += [(tokens[0], '1')]
  81   return result
  82
  83
  84 def GetGypVars(supplemental_files):
  85   """Returns a dictionary of all GYP vars."""
  86   # Find the .gyp directory in the user's home directory.
  87   home_dot_gyp = os.environ.get('GYP_CONFIG_DIR', None)
  88   if home_dot_gyp:
  89     home_dot_gyp = os.path.expanduser(home_dot_gyp)
  90   if not home_dot_gyp:
  91     home_vars = ['HOME']
  92     if sys.platform in ('cygwin', 'win32'):
  93       home_vars.append('USERPROFILE')
  94     for home_var in home_vars:
  95       home = os.getenv(home_var)
  96       if home != None:
  97         home_dot_gyp = os.path.join(home, '.gyp')
  98         if not os.path.exists(home_dot_gyp):
  99           home_dot_gyp = None
 100         else:
 101           break
 102
 103   if home_dot_gyp:
 104     include_gypi = os.path.join(home_dot_gyp, "include.gypi")
 105     if os.path.exists(include_gypi):
 106       supplemental_files += [include_gypi]
 107
 108   # GYP defines from the supplemental.gypi files.
 109   supp_items = []
 110   for supplement in supplemental_files:
 111     with open(supplement, 'r') as f:
 112       try:
 113         file_data = eval(f.read(), {'__builtins__': None}, None)
 114       except SyntaxError, e:
 115         e.filename = os.path.abspath(supplement)
 116         raise
 117       variables = file_data.get('variables', [])
 118       for v in variables:
 119         supp_items += [(v, str(variables[v]))]
 120
 121   # GYP defines from the environment.
 122   env_items = ProcessGypDefinesItems(
 123       shlex.split(os.environ.get('GYP_DEFINES', '')))
 124
 125   # GYP defines from the command line. We can't use optparse since we want
 126   # to ignore all arguments other than "-D".
 127   cmdline_input_items = []
 128   for i in range(len(sys.argv))[1:]:
 129     if sys.argv[i].startswith('-D'):
 130       if sys.argv[i] == '-D' and i + 1 < len(sys.argv):
 131         cmdline_input_items += [sys.argv[i + 1]]
 132       elif len(sys.argv[i]) > 2:
 133         cmdline_input_items += [sys.argv[i][2:]]
 134   cmdline_items = ProcessGypDefinesItems(cmdline_input_items)
 135
 136   vars_dict = dict(supp_items + env_items + cmdline_items)
 137   return vars_dict
 138
 139
 140 def GetOutputDirectory():
 141   """Returns the output directory that GYP will use."""
 142   # GYP generator flags from the command line. We can't use optparse since we
 143   # want to ignore all arguments other than "-G".
 144   needle = '-Goutput_dir='
 145   cmdline_input_items = []
 146   for item in sys.argv[1:]:
 147     if item.startswith(needle):
 148       return item[len(needle):]
 149
 150   env_items = shlex.split(os.environ.get('GYP_GENERATOR_FLAGS', ''))
 151   needle = 'output_dir='
 152   for item in env_items:
 153     if item.startswith(needle):
 154       return item[len(needle):]
 155
 156   return "out"
 157
 158
 159 def additional_include_files(supplemental_files, args=[]):
 160   """
 161   Returns a list of additional (.gypi) files to include, without duplicating
 162   ones that are already specified on the command line. The list of supplemental
 163   include files is passed in as an argument.
 164   """
 165   # Determine the include files specified on the command line.
 166   # This doesn't cover all the different option formats you can use,
 167   # but it's mainly intended to avoid duplicating flags on the automatic
 168   # makefile regeneration which only uses this format.
 169   specified_includes = set()
 170   for arg in args:
 171     if arg.startswith('-I') and len(arg) > 2:
 172       specified_includes.add(os.path.realpath(arg[2:]))
 173
 174   result = []
 175   def AddInclude(path):
 176     if os.path.realpath(path) not in specified_includes:
 177       result.append(path)
 178
 179   # Always include common.gypi.
 180   AddInclude(os.path.join(script_dir, 'common.gypi'))
 181
 182   # Optionally add supplemental .gypi files if present.
 183   for supplement in supplemental_files:
 184     AddInclude(supplement)
 185
 186   return result
 187
 188
 189 if __name__ == '__main__':
 190   args = sys.argv[1:]
 191
 192   # TODO(sky): remove analyzer2 once updated recipes.
 193   use_analyzer = len(args) and args[0] == '--analyzer'
 194   if use_analyzer:
 195     args.pop(0)
 196     os.environ['GYP_GENERATORS'] = 'analyzer'
 197     args.append('-Gfile_path=' + args.pop(0))
 198   elif len(args) and args[0] == '--analyzer2':
 199     use_analyzer = True
 200     args.pop(0)
 201     os.environ['GYP_GENERATORS'] = 'analyzer'
 202     args.append('-Gconfig_path=' + args.pop(0))
 203     args.append('-Ganalyzer_output_path=' + args.pop(0))
 204
 205   if int(os.environ.get('GYP_CHROMIUM_NO_ACTION', 0)):
 206     # Check for landmines (reasons to clobber the build) in any case.
 207     print 'Running build/landmines.py...'
 208     subprocess.check_call(
 209         [sys.executable, os.path.join(script_dir, 'landmines.py')])
 210     print 'Skipping gyp_chromium due to GYP_CHROMIUM_NO_ACTION env var.'
 211     sys.exit(0)
 212
 213   # Use the Psyco JIT if available.
 214   if psyco:
 215     psyco.profile()
 216     print "Enabled Psyco JIT."
 217
 218   # Fall back on hermetic python if we happen to get run under cygwin.
 219   # TODO(bradnelson): take this out once this issue is fixed:
 220   #    http://code.google.com/p/gyp/issues/detail?id=177
 221   if sys.platform == 'cygwin':
 222     import find_depot_tools
 223     depot_tools_path = find_depot_tools.add_depot_tools_to_path()
 224     python_dir = sorted(glob.glob(os.path.join(depot_tools_path,
 225                                                'python2*_bin')))[-1]
 226     env = os.environ.copy()
 227     env['PATH'] = python_dir + os.pathsep + env.get('PATH', '')
 228     p = subprocess.Popen(
 229        [os.path.join(python_dir, 'python.exe')] + sys.argv,
 230        env=env, shell=False)
 231     p.communicate()
 232     sys.exit(p.returncode)
 233
 234   gyp_helper.apply_chromium_gyp_env()
 235
 236   # This could give false positives since it doesn't actually do real option
 237   # parsing.  Oh well.
 238   gyp_file_specified = False
 239   for arg in args:
 240     if arg.endswith('.gyp'):
 241       gyp_file_specified = True
 242       break
 243
 244   # If we didn't get a file, check an env var, and then fall back to
 245   # assuming 'all.gyp' from the same directory as the script.
 246   if not gyp_file_specified:
 247     gyp_file = os.environ.get('CHROMIUM_GYP_FILE')
 248     if gyp_file:
 249       # Note that CHROMIUM_GYP_FILE values can't have backslashes as
 250       # path separators even on Windows due to the use of shlex.split().
 251       args.extend(shlex.split(gyp_file))
 252     else:
 253       args.append(os.path.join(script_dir, 'all.gyp'))
 254
 255   # There shouldn't be a circular dependency relationship between .gyp files,
 256   # but in Chromium's .gyp files, on non-Mac platforms, circular relationships
 257   # currently exist.  The check for circular dependencies is currently
 258   # bypassed on other platforms, but is left enabled on the Mac, where a
 259   # violation of the rule causes Xcode to misbehave badly.
 260   # TODO(mark): Find and kill remaining circular dependencies, and remove this
 261   # option.  http://crbug.com/35878.
 262   # TODO(tc): Fix circular dependencies in ChromiumOS then add linux2 to the
 263   # list.
 264   if sys.platform not in ('darwin',):
 265     args.append('--no-circular-check')
 266
 267   # We explicitly don't support the make gyp generator (crbug.com/348686). Be
 268   # nice and fail here, rather than choking in gyp.
 269   if re.search(r'(^|,|\s)make($|,|\s)', os.environ.get('GYP_GENERATORS', '')):
 270     print 'Error: make gyp generator not supported (check GYP_GENERATORS).'
 271     sys.exit(1)
 272
 273   # Default to ninja on linux and windows, but only if no generator has
 274   # explicitly been set.
 275   # Also default to ninja on mac, but only when not building chrome/ios.
 276   # . -f / --format has precedence over the env var, no need to check for it
 277   # . set the env var only if it hasn't been set yet
 278   # . chromium.gyp_env has been applied to os.environ at this point already
 279   if sys.platform.startswith(('linux', 'win', 'freebsd')) and \
 280       not os.environ.get('GYP_GENERATORS'):
 281     os.environ['GYP_GENERATORS'] = 'ninja'
 282   elif sys.platform == 'darwin' and not os.environ.get('GYP_GENERATORS') and \
 283       not 'OS=ios' in os.environ.get('GYP_DEFINES', []):
 284     os.environ['GYP_GENERATORS'] = 'ninja'
 285
 286   vs2013_runtime_dll_dirs = vs_toolchain.SetEnvironmentAndGetRuntimeDllDirs()
 287
 288   # If CHROMIUM_GYP_SYNTAX_CHECK is set to 1, it will invoke gyp with --check
 289   # to enfore syntax checking.
 290   syntax_check = os.environ.get('CHROMIUM_GYP_SYNTAX_CHECK')
 291   if syntax_check and int(syntax_check):
 292     args.append('--check')
 293
 294   supplemental_includes = GetSupplementalFiles()
 295   gyp_vars_dict = GetGypVars(supplemental_includes)
 296
 297   # TODO(dmikurube): Remove these checks and messages after a while.
 298   if ('linux_use_tcmalloc' in gyp_vars_dict or
 299       'android_use_tcmalloc' in gyp_vars_dict):
 300     print '*****************************************************************'
 301     print '"linux_use_tcmalloc" and "android_use_tcmalloc" are deprecated!'
 302     print '-----------------------------------------------------------------'
 303     print 'You specify "linux_use_tcmalloc" or "android_use_tcmalloc" in'
 304     print 'your GYP_DEFINES. Please switch them into "use_allocator" now.'
 305     print 'See http://crbug.com/345554 for the details.'
 306     print '*****************************************************************'
 307
 308   # Automatically turn on crosscompile support for platforms that need it.
 309   # (The Chrome OS build sets CC_host / CC_target which implicitly enables
 310   # this mode.)
 311   if all(('ninja' in os.environ.get('GYP_GENERATORS', ''),
 312           gyp_vars_dict.get('OS') in ['android', 'ios'],
 313           'GYP_CROSSCOMPILE' not in os.environ)):
 314     os.environ['GYP_CROSSCOMPILE'] = '1'
 315   if gyp_vars_dict.get('OS') == 'android':
 316     args.append('--check')
 317
 318   args.extend(
 319       ['-I' + i for i in additional_include_files(supplemental_includes, args)])
 320
 321   args.extend(['-D', 'gyp_output_dir=' + GetOutputDirectory()])
 322
 323   if not use_analyzer:
 324     print 'Updating projects from gyp files...'
 325     sys.stdout.flush()
 326
 327   # Off we go...
 328   gyp_rc = gyp.main(args)
 329
 330   if not use_analyzer:
 331     # Check for landmines (reasons to clobber the build). This must be run here,
 332     # rather than a separate runhooks step so that any environment modifications
 333     # from above are picked up.
 334     print 'Running build/landmines.py...'
 335     subprocess.check_call(
 336       [sys.executable, os.path.join(script_dir, 'landmines.py')])
 337
 338     if vs2013_runtime_dll_dirs:
 339       x64_runtime, x86_runtime = vs2013_runtime_dll_dirs
 340       vs_toolchain.CopyVsRuntimeDlls(
 341         os.path.join(chrome_src, GetOutputDirectory()),
 342         (x86_runtime, x64_runtime))
 343
 344   sys.exit(gyp_rc)