third_party/instrumented_libraries/download_build_install.py

   1 #!/usr/bin/python
   2 # Copyright 2013 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """Downloads, builds (with instrumentation) and installs shared libraries."""
   7
   8 import argparse
   9 import os
  10 import platform
  11 import shlex
  12 import shutil
  13 import subprocess
  14 import sys
  15
  16 class ScopedChangeDirectory(object):
  17   """Changes current working directory and restores it back automatically."""
  18
  19   def __init__(self, path):
  20     self.path = path
  21     self.old_path = ''
  22
  23   def __enter__(self):
  24     self.old_path = os.getcwd()
  25     os.chdir(self.path)
  26     return self
  27
  28   def __exit__(self, exc_type, exc_value, traceback):
  29     os.chdir(self.old_path)
  30
  31
  32 def get_script_absolute_path():
  33   return os.path.dirname(os.path.abspath(__file__))
  34
  35
  36 def get_package_build_dependencies(package):
  37   command = 'apt-get -s build-dep %s | grep Inst | cut -d " " -f 2' % package
  38   command_result = subprocess.Popen(command, stdout=subprocess.PIPE,
  39                                     shell=True)
  40   if command_result.wait():
  41     raise Exception('Failed to determine build dependencies for %s' % package)
  42   build_dependencies = [l.strip() for l in command_result.stdout]
  43   return build_dependencies
  44
  45
  46 def check_package_build_dependencies(package):
  47   build_dependencies = get_package_build_dependencies(package)
  48   if len(build_dependencies):
  49     print >> sys.stderr, 'Please, install build-dependencies for %s' % package
  50     print >> sys.stderr, 'One-liner for APT:'
  51     print >> sys.stderr, 'sudo apt-get -y --no-remove build-dep %s' % package
  52     sys.exit(1)
  53
  54
  55 def shell_call(command, verbose=False, environment=None):
  56   """ Wrapper on subprocess.Popen
  57
  58   Calls command with specific environment and verbosity using
  59   subprocess.Popen
  60
  61   Args:
  62     command: Command to run in shell.
  63     verbose: If False, hides all stdout and stderr in case of successful build.
  64         Otherwise, always prints stdout and stderr.
  65     environment: Parameter 'env' for subprocess.Popen.
  66
  67   Returns:
  68     None
  69
  70   Raises:
  71     Exception: if return code after call is not zero.
  72   """
  73   child = subprocess.Popen(
  74       command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
  75       env=environment, shell=True)
  76   stdout, stderr = child.communicate()
  77   if verbose or child.returncode:
  78     print stdout
  79   if child.returncode:
  80     raise Exception('Failed to run: %s' % command)
  81
  82
  83 def run_shell_commands(commands, verbose=False, environment=None):
  84   for command in commands:
  85     shell_call(command, verbose, environment)
  86
  87
  88 def destdir_configure_make_install(parsed_arguments, environment,
  89                                    install_prefix):
  90   configure_command = './configure %s' % parsed_arguments.extra_configure_flags
  91   configure_command += ' --libdir=/lib/'
  92   # Installing to a temporary directory allows us to safely clean up the .la
  93   # files below.
  94   destdir = '%s/debian/instrumented_build' % os.getcwd()
  95   # Some makefiles use BUILDROOT instead of DESTDIR.
  96   make_command = 'make DESTDIR=%s BUILDROOT=%s' % (destdir, destdir)
  97   run_shell_commands([
  98       configure_command,
  99       '%s -j%s' % (make_command, parsed_arguments.jobs),
 100       # Parallel install is flaky for some packages.
 101       '%s install -j1' % make_command,
 102       # Kill the .la files. They contain absolute paths, and will cause build
 103       # errors in dependent libraries.
 104       'rm %s/lib/*.la -f' % destdir,
 105       # Now move the contents of the temporary destdir to their final place.
 106       'cp %s/* %s/ -rdf' % (destdir, install_prefix)],
 107                      parsed_arguments.verbose, environment)
 108
 109
 110 def nss_make_and_copy(parsed_arguments, environment, install_prefix):
 111   # NSS uses a build system that's different from configure/make/install. All
 112   # flags must be passed as arguments to make.
 113   make_args = []
 114   # Do an optimized build.
 115   make_args.append('BUILD_OPT=1')
 116   # Set USE_64=1 on x86_64 systems.
 117   if platform.architecture()[0] == '64bit':
 118     make_args.append('USE_64=1')
 119   # Passing C(XX)FLAGS overrides the defaults, and EXTRA_C(XX)FLAGS is not
 120   # supported. Append our extra flags to CC/CXX.
 121   make_args.append('CC="%s %s"' % (environment['CC'], environment['CFLAGS']))
 122   make_args.append('CXX="%s %s"' %
 123                    (environment['CXX'], environment['CXXFLAGS']))
 124   # We need to override ZDEFS_FLAG at least to prevent -Wl,-z,defs.
 125   # Might as well use this to pass the linker flags, since ZDEF_FLAG is always
 126   # added during linking on Linux.
 127   make_args.append('ZDEFS_FLAG="-Wl,-z,nodefs %s"' % environment['LDFLAGS'])
 128   make_args.append('NSPR_INCLUDE_DIR=/usr/include/nspr')
 129   make_args.append('NSPR_LIB_DIR=%s/lib' % install_prefix)
 130   make_args.append('NSS_ENABLE_ECC=1')
 131   # Make sure we don't override the default flags.
 132   for variable in ['CFLAGS', 'CXXFLAGS', 'LDFLAGS']:
 133     del environment[variable]
 134   with ScopedChangeDirectory('nss') as cd_nss:
 135     # -j is not supported
 136     shell_call('make %s' % ' '.join(make_args), parsed_arguments.verbose,
 137                environment)
 138     # 'make install' is not supported. Copy the DSOs manually.
 139     install_dir = '%s/lib/' % install_prefix
 140     for (dirpath, dirnames, filenames) in os.walk('./lib/'):
 141       for filename in filenames:
 142         if filename.endswith('.so'):
 143           full_path = os.path.join(dirpath, filename)
 144           if parsed_arguments.verbose:
 145             print 'download_build_install.py: installing %s' % full_path
 146           shutil.copy(full_path, install_dir)
 147
 148
 149 def libcap2_make_install(parsed_arguments, environment, install_prefix):
 150   # libcap2 doesn't come with a configure script
 151   make_args = [
 152       '%s="%s"' % (name, environment[name])
 153       for name in['CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS']]
 154   shell_call('make -j%s %s' % (parsed_arguments.jobs, ' '.join(make_args)),
 155              parsed_arguments.verbose, environment)
 156   install_args = [
 157       'DESTDIR=%s' % install_prefix,
 158       # Do not install in lib64/.
 159       'lib=lib',
 160       # Skip a step that requires sudo.
 161       'RAISE_SETFCAP=no'
 162   ]
 163   shell_call('make -j%s install %s' %
 164              (parsed_arguments.jobs, ' '.join(install_args)),
 165              parsed_arguments.verbose, environment)
 166
 167
 168 def libpci3_make_install(parsed_arguments, environment, install_prefix):
 169   # pciutils doesn't have a configure script
 170   # This build script follows debian/rules.
 171
 172   # `make install' will create a "$(DESTDIR)-udeb" directory alongside destdir.
 173   # We don't want that in our product dir, so we use an intermediate directory.
 174   destdir = '%s/debian/pciutils' % os.getcwd()
 175   make_args = [
 176       '%s="%s"' % (name, environment[name])
 177       for name in['CC', 'CXX', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS']]
 178   make_args.append('SHARED=yes')
 179   paths = [
 180       'LIBDIR=/lib/',
 181       'PREFIX=/usr',
 182       'SBINDIR=/usr/bin',
 183       'IDSDIR=/usr/share/misc',
 184   ]
 185   install_args = ['DESTDIR=%s' % destdir]
 186   run_shell_commands([
 187       'mkdir -p %s-udeb/usr/bin' % destdir,
 188       'make -j%s %s' % (parsed_arguments.jobs, ' '.join(make_args + paths)),
 189       'make -j%s %s install' % (
 190           parsed_arguments.jobs,
 191           ' '.join(install_args + paths))],
 192                      parsed_arguments.verbose, environment)
 193   # Now move the contents of the temporary destdir to their final place.
 194   run_shell_commands([
 195       'cp %s/* %s/ -rd' % (destdir, install_prefix),
 196       'install -m 644 lib/libpci.so* %s/lib/' % install_prefix,
 197       'ln -sf libpci.so.3.1.8 %s/lib/libpci.so.3' % install_prefix],
 198                      parsed_arguments.verbose, environment)
 199
 200
 201 def build_and_install(parsed_arguments, environment, install_prefix):
 202   if parsed_arguments.build_method == 'destdir':
 203     destdir_configure_make_install(
 204         parsed_arguments, environment, install_prefix)
 205   elif parsed_arguments.build_method == 'custom_nss':
 206     nss_make_and_copy(parsed_arguments, environment, install_prefix)
 207   elif parsed_arguments.build_method == 'custom_libcap':
 208     libcap2_make_install(parsed_arguments, environment, install_prefix)
 209   elif parsed_arguments.build_method == 'custom_libpci3':
 210     libpci3_make_install(parsed_arguments, environment, install_prefix)
 211   else:
 212     raise Exception('Unrecognized build method: %s' %
 213                     parsed_arguments.build_method)
 214
 215
 216 def unescape_flags(s):
 217   # GYP escapes the build flags as if they are going to be inserted directly
 218   # into the command line. Since we pass them via CFLAGS/LDFLAGS, we must drop
 219   # the double quotes accordingly.
 220   return ' '.join(shlex.split(s))
 221
 222
 223 def build_environment(parsed_arguments, product_directory, install_prefix):
 224   environment = os.environ.copy()
 225   # The CC/CXX environment variables take precedence over the command line
 226   # flags.
 227   if 'CC' not in environment and parsed_arguments.cc:
 228     environment['CC'] = parsed_arguments.cc
 229   if 'CXX' not in environment and parsed_arguments.cxx:
 230     environment['CXX'] = parsed_arguments.cxx
 231
 232   cflags = unescape_flags(parsed_arguments.cflags)
 233   if parsed_arguments.sanitizer_blacklist:
 234     cflags += ' -fsanitize-blacklist=%s/%s' % (
 235         get_script_absolute_path(),
 236         parsed_arguments.sanitizer_blacklist)
 237   environment['CFLAGS'] = cflags
 238   environment['CXXFLAGS'] = cflags
 239
 240   ldflags = unescape_flags(parsed_arguments.ldflags)
 241   # Make sure the linker searches the instrumented libraries dir for
 242   # library dependencies.
 243   environment['LDFLAGS'] = '%s -L%s/lib' % (ldflags, install_prefix)
 244
 245   if parsed_arguments.sanitizer_type == 'asan':
 246     # Do not report leaks during the build process.
 247     environment['ASAN_OPTIONS'] = '%s:detect_leaks=0' % \
 248         environment.get('ASAN_OPTIONS', '')
 249
 250   # libappindicator1 needs this.
 251   environment['CSC'] = '/usr/bin/mono-csc'
 252   return environment
 253
 254
 255
 256 def download_build_install(parsed_arguments):
 257   product_directory = os.path.normpath('%s/%s' % (
 258       get_script_absolute_path(),
 259       parsed_arguments.product_directory))
 260
 261   install_prefix = '%s/instrumented_libraries/%s' % (
 262       product_directory,
 263       parsed_arguments.sanitizer_type)
 264
 265   environment = build_environment(parsed_arguments, product_directory,
 266                                   install_prefix)
 267
 268   package_directory = '%s/%s' % (parsed_arguments.intermediate_directory,
 269                                  parsed_arguments.package)
 270
 271   # Clobber by default, unless the developer wants to hack on the package's
 272   # source code.
 273   clobber = (environment.get('INSTRUMENTED_LIBRARIES_NO_CLOBBER', '') != '1')
 274
 275   download_source = True
 276   if os.path.exists(package_directory):
 277     if clobber:
 278       shell_call('rm -rf %s' % package_directory, parsed_arguments.verbose)
 279     else:
 280       download_source = False
 281   if download_source:
 282     os.makedirs(package_directory)
 283
 284   with ScopedChangeDirectory(package_directory) as cd_package:
 285     if download_source:
 286       shell_call('apt-get source %s' % parsed_arguments.package,
 287                  parsed_arguments.verbose)
 288     # There should be exactly one subdirectory after downloading a package.
 289     subdirectories = [d for d in os.listdir('.') if os.path.isdir(d)]
 290     if len(subdirectories) != 1:
 291       raise Exception('apt-get source %s must create exactly one subdirectory.'
 292          % parsed_arguments.package)
 293     with ScopedChangeDirectory(subdirectories[0]):
 294       # Here we are in the package directory.
 295       if download_source:
 296         # Patch/run_before_build steps are only done once.
 297         if parsed_arguments.patch:
 298           shell_call(
 299               'patch -p1 -i %s/%s' %
 300               (os.path.relpath(cd_package.old_path),
 301                parsed_arguments.patch),
 302               parsed_arguments.verbose)
 303         if parsed_arguments.run_before_build:
 304           shell_call(
 305               '%s/%s' %
 306               (os.path.relpath(cd_package.old_path),
 307                parsed_arguments.run_before_build),
 308               parsed_arguments.verbose)
 309       try:
 310         build_and_install(parsed_arguments, environment, install_prefix)
 311       except Exception as exception:
 312         print exception
 313         print 'Failed to build package %s.' % parsed_arguments.package
 314         print ('Probably, some of its dependencies are not installed: %s' %
 315                ' '.join(get_package_build_dependencies(parsed_arguments.package)))
 316         sys.exit(1)
 317
 318   # Touch a txt file to indicate package is installed.
 319   open('%s/%s.txt' % (install_prefix, parsed_arguments.package), 'w').close()
 320
 321   # Remove downloaded package and generated temporary build files.
 322   # Failed builds intentionally skip this step, in order to aid in tracking down
 323   # build failures.
 324   if clobber:
 325     shell_call('rm -rf %s' % package_directory, parsed_arguments.verbose)
 326
 327 def main():
 328   argument_parser = argparse.ArgumentParser(
 329       description='Download, build and install instrumented package')
 330
 331   argument_parser.add_argument('-j', '--jobs', type=int, default=1)
 332   argument_parser.add_argument('-p', '--package', required=True)
 333   argument_parser.add_argument(
 334       '-i', '--product-directory', default='.',
 335       help='Relative path to the directory with chrome binaries')
 336   argument_parser.add_argument(
 337       '-m', '--intermediate-directory', default='.',
 338       help='Relative path to the directory for temporary build files')
 339   argument_parser.add_argument('--extra-configure-flags', default='')
 340   argument_parser.add_argument('--cflags', default='')
 341   argument_parser.add_argument('--ldflags', default='')
 342   argument_parser.add_argument('-s', '--sanitizer-type', required=True,
 343                                choices=['asan', 'msan', 'tsan'])
 344   argument_parser.add_argument('-v', '--verbose', action='store_true')
 345   argument_parser.add_argument('--check-build-deps', action='store_true')
 346   argument_parser.add_argument('--cc')
 347   argument_parser.add_argument('--cxx')
 348   argument_parser.add_argument('--patch', default='')
 349   # This should be a shell script to run before building specific libraries.
 350   # This will be run after applying the patch above.
 351   argument_parser.add_argument('--run-before-build', default='')
 352   argument_parser.add_argument('--build-method', default='destdir')
 353   argument_parser.add_argument('--sanitizer-blacklist', default='')
 354
 355   # Ignore all empty arguments because in several cases gyp passes them to the
 356   # script, but ArgumentParser treats them as positional arguments instead of
 357   # ignoring (and doesn't have such options).
 358   parsed_arguments = argument_parser.parse_args(
 359       [arg for arg in sys.argv[1:] if len(arg) != 0])
 360   # Ensure current working directory is this script directory.
 361   os.chdir(get_script_absolute_path())
 362   # Ensure all build dependencies are installed.
 363   if parsed_arguments.check_build_deps:
 364     check_package_build_dependencies(parsed_arguments.package)
 365
 366   download_build_install(parsed_arguments)
 367
 368
 369 if __name__ == '__main__':
 370   main()