tools/export_tarball/export_tarball.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """
   7 This tool creates a tarball with all the sources, but without .svn directories.
   8
   9 It can also remove files which are not strictly required for build, so that
  10 the resulting tarball can be reasonably small (last time it was ~110 MB).
  11
  12 Example usage:
  13
  14 export_tarball.py /foo/bar
  15
  16 The above will create file /foo/bar.tar.bz2.
  17 """
  18
  19 import optparse
  20 import os
  21 import subprocess
  22 import sys
  23 import tarfile
  24
  25
  26 NONESSENTIAL_DIRS = (
  27     'breakpad/src/processor/testdata',
  28     'chrome/browser/resources/tracing/tests',
  29     'chrome/common/extensions/docs',
  30     'chrome/tools/test/reference_build',
  31     'courgette/testdata',
  32     'data',
  33     'native_client/src/trusted/service_runtime/testdata',
  34     'src/chrome/test/data',
  35     'o3d/documentation',
  36     'o3d/samples',
  37     'o3d/tests',
  38     'ppapi/examples',
  39     'ppapi/native_client/tests',
  40     'third_party/angle/samples/gles2_book',
  41     'third_party/findbugs',
  42     'third_party/hunspell_dictionaries',
  43     'third_party/hunspell/tests',
  44     'third_party/lighttpd',
  45     'third_party/sqlite/src/test',
  46     'third_party/sqlite/test',
  47     'third_party/vc_80',
  48     'third_party/xdg-utils/tests',
  49     'third_party/yasm/source/patched-yasm/modules/arch/x86/tests',
  50     'third_party/yasm/source/patched-yasm/modules/dbgfmts/dwarf2/tests',
  51     'third_party/yasm/source/patched-yasm/modules/objfmts/bin/tests',
  52     'third_party/yasm/source/patched-yasm/modules/objfmts/coff/tests',
  53     'third_party/yasm/source/patched-yasm/modules/objfmts/elf/tests',
  54     'third_party/yasm/source/patched-yasm/modules/objfmts/macho/tests',
  55     'third_party/yasm/source/patched-yasm/modules/objfmts/rdf/tests',
  56     'third_party/yasm/source/patched-yasm/modules/objfmts/win32/tests',
  57     'third_party/yasm/source/patched-yasm/modules/objfmts/win64/tests',
  58     'third_party/yasm/source/patched-yasm/modules/objfmts/xdf/tests',
  59     'third_party/WebKit/LayoutTests',
  60     'third_party/WebKit/Source/JavaScriptCore/tests',
  61     'third_party/WebKit/Source/WebCore/ChangeLog',
  62     'third_party/WebKit/Source/WebKit2',
  63     'third_party/WebKit/Tools/Scripts',
  64     'tools/gyp/test',
  65     'v8/test',
  66     'webkit/data/layout_tests',
  67     'webkit/tools/test/reference_build',
  68 )
  69
  70 TESTDIRS = (
  71     'chrome/test/data',
  72     'content/test/data',
  73     'media/test/data',
  74     'net/data',
  75 )
  76
  77
  78 def GetSourceDirectory():
  79   return os.path.realpath(
  80     os.path.join(os.path.dirname(__file__), '..', '..', '..', 'src'))
  81
  82
  83 # Workaround lack of the exclude parameter in add method in python-2.4.
  84 # TODO(phajdan.jr): remove the workaround when it's not needed on the bot.
  85 class MyTarFile(tarfile.TarFile):
  86   def set_remove_nonessential_files(self, remove):
  87     self.__remove_nonessential_files = remove
  88
  89   def set_verbose(self, verbose):
  90     self.__verbose = verbose
  91
  92   def __report_skipped(self, name):
  93     if self.__verbose:
  94       print 'D\t%s' % name
  95
  96   def __report_added(self, name):
  97     if self.__verbose:
  98       print 'A\t%s' % name
  99
 100   def add(self, name, arcname=None, recursive=True, exclude=None, filter=None):
 101     head, tail = os.path.split(name)
 102     if tail in ('.svn', '.git'):
 103       self.__report_skipped(name)
 104       return
 105
 106     if self.__remove_nonessential_files:
 107       # WebKit change logs take quite a lot of space. This saves ~10 MB
 108       # in a bzip2-compressed tarball.
 109       if 'ChangeLog' in name:
 110         self.__report_skipped(name)
 111         return
 112
 113       # Remove contents of non-essential directories, but preserve gyp files,
 114       # so that build/gyp_chromium can work.
 115       for nonessential_dir in (NONESSENTIAL_DIRS + TESTDIRS):
 116         dir_path = os.path.join(GetSourceDirectory(), nonessential_dir)
 117         if (name.startswith(dir_path) and
 118             os.path.isfile(name) and
 119             'gyp' not in name):
 120           self.__report_skipped(name)
 121           return
 122
 123     self.__report_added(name)
 124     tarfile.TarFile.add(self, name, arcname=arcname, recursive=recursive)
 125
 126
 127 def main(argv):
 128   parser = optparse.OptionParser()
 129   parser.add_option("--basename")
 130   parser.add_option("--remove-nonessential-files",
 131                     dest="remove_nonessential_files",
 132                     action="store_true", default=False)
 133   parser.add_option("--test-data", action="store_true")
 134   # TODO(phajdan.jr): Remove --xz option when it's not needed for compatibility.
 135   parser.add_option("--xz", action="store_true")
 136   parser.add_option("--verbose", action="store_true", default=False)
 137   parser.add_option("--progress", action="store_true", default=False)
 138
 139   options, args = parser.parse_args(argv)
 140
 141   if len(args) != 1:
 142     print 'You must provide only one argument: output file name'
 143     print '(without .tar.xz extension).'
 144     return 1
 145
 146   if not os.path.exists(GetSourceDirectory()):
 147     print 'Cannot find the src directory ' + GetSourceDirectory()
 148     return 1
 149
 150   # These two commands are from src/DEPS; please keep them in sync.
 151   if subprocess.call(['python', 'build/util/lastchange.py', '-o',
 152                       'build/util/LASTCHANGE'], cwd=GetSourceDirectory()) != 0:
 153     print 'Could not run build/util/lastchange.py to update LASTCHANGE.'
 154     return 1
 155   if subprocess.call(['python', 'build/util/lastchange.py', '-s',
 156                       'third_party/WebKit', '-o',
 157                       'build/util/LASTCHANGE.blink'],
 158                      cwd=GetSourceDirectory()) != 0:
 159     print 'Could not run build/util/lastchange.py to update LASTCHANGE.blink.'
 160     return 1
 161
 162   output_fullname = args[0] + '.tar'
 163   output_basename = options.basename or os.path.basename(args[0])
 164
 165   archive = MyTarFile.open(output_fullname, 'w')
 166   archive.set_remove_nonessential_files(options.remove_nonessential_files)
 167   archive.set_verbose(options.verbose)
 168   try:
 169     if options.test_data:
 170       for directory in TESTDIRS:
 171         archive.add(os.path.join(GetSourceDirectory(), directory),
 172                     arcname=os.path.join(output_basename, directory))
 173     else:
 174       archive.add(GetSourceDirectory(), arcname=output_basename)
 175   finally:
 176     archive.close()
 177
 178   if options.progress:
 179     sys.stdout.flush()
 180     pv = subprocess.Popen(
 181         ['pv', '--force', output_fullname],
 182         stdout=subprocess.PIPE,
 183         stderr=sys.stdout)
 184     with open(output_fullname + '.xz', 'w') as f:
 185       rc = subprocess.call(['xz', '-9', '-'], stdin=pv.stdout, stdout=f)
 186     pv.wait()
 187   else:
 188     rc = subprocess.call(['xz', '-9', output_fullname])
 189
 190   if rc != 0:
 191     print 'xz -9 failed!'
 192     return 1
 193
 194   return 0
 195
 196
 197 if __name__ == "__main__":
 198   sys.exit(main(sys.argv[1:]))