tools/export_tarball/export_tarball.py

   1 #!/usr/bin/env python
   2 # Copyright (c) 2012 The Chromium Authors. All rights reserved.
   3 # Use of this source code is governed by a BSD-style license that can be
   4 # found in the LICENSE file.
   5
   6 """
   7 This tool creates a tarball with all the sources, but without .svn directories.
   8
   9 It can also remove files which are not strictly required for build, so that
  10 the resulting tarball can be reasonably small (last time it was ~110 MB).
  11
  12 Example usage:
  13
  14 export_tarball.py /foo/bar
  15
  16 The above will create file /foo/bar.tar.bz2.
  17 """
  18
  19 import optparse
  20 import os
  21 import subprocess
  22 import sys
  23 import tarfile
  24
  25
  26 NONESSENTIAL_DIRS = (
  27     'breakpad/src/processor/testdata',
  28     'chrome/browser/resources/tracing/tests',
  29     'chrome/common/extensions/docs',
  30     'chrome/tools/test/reference_build',
  31     'courgette/testdata',
  32     'data',
  33     'native_client/src/trusted/service_runtime/testdata',
  34     'src/chrome/test/data',
  35     'o3d/documentation',
  36     'o3d/samples',
  37     'o3d/tests',
  38     'ppapi/examples',
  39     'ppapi/native_client/tests',
  40     'third_party/angle/samples/gles2_book',
  41     'third_party/findbugs',
  42     'third_party/hunspell_dictionaries',
  43     'third_party/hunspell/tests',
  44     'third_party/lighttpd',
  45     'third_party/sqlite/src/test',
  46     'third_party/sqlite/test',
  47     'third_party/vc_80',
  48     'third_party/xdg-utils/tests',
  49     'third_party/yasm/source/patched-yasm/modules/arch/x86/tests',
  50     'third_party/yasm/source/patched-yasm/modules/dbgfmts/dwarf2/tests',
  51     'third_party/yasm/source/patched-yasm/modules/objfmts/bin/tests',
  52     'third_party/yasm/source/patched-yasm/modules/objfmts/coff/tests',
  53     'third_party/yasm/source/patched-yasm/modules/objfmts/elf/tests',
  54     'third_party/yasm/source/patched-yasm/modules/objfmts/macho/tests',
  55     'third_party/yasm/source/patched-yasm/modules/objfmts/rdf/tests',
  56     'third_party/yasm/source/patched-yasm/modules/objfmts/win32/tests',
  57     'third_party/yasm/source/patched-yasm/modules/objfmts/win64/tests',
  58     'third_party/yasm/source/patched-yasm/modules/objfmts/xdf/tests',
  59     'third_party/WebKit/LayoutTests',
  60     'third_party/WebKit/Source/JavaScriptCore/tests',
  61     'third_party/WebKit/Source/WebCore/ChangeLog',
  62     'third_party/WebKit/Source/WebKit2',
  63     'third_party/WebKit/Tools/Scripts',
  64     'tools/gyp/test',
  65     'v8/test',
  66     'webkit/data/layout_tests',
  67     'webkit/tools/test/reference_build',
  68 )
  69
  70 TESTDIRS = (
  71     'chrome/test/data',
  72     'content/test/data',
  73     'media/test/data',
  74     'net/data',
  75 )
  76
  77
  78 def GetSourceDirectory():
  79   return os.path.realpath(
  80     os.path.join(os.path.dirname(__file__), '..', '..', '..', 'src'))
  81
  82
  83 # Workaround lack of the exclude parameter in add method in python-2.4.
  84 # TODO(phajdan.jr): remove the workaround when it's not needed on the bot.
  85 class MyTarFile(tarfile.TarFile):
  86   def set_remove_nonessential_files(self, remove):
  87     self.__remove_nonessential_files = remove
  88
  89   def add(self, name, arcname=None, recursive=True, exclude=None, filter=None):
  90     head, tail = os.path.split(name)
  91     if tail in ('.svn', '.git'):
  92       return
  93
  94     if self.__remove_nonessential_files:
  95       # WebKit change logs take quite a lot of space. This saves ~10 MB
  96       # in a bzip2-compressed tarball.
  97       if 'ChangeLog' in name:
  98         return
  99
 100       # Remove contents of non-essential directories, but preserve gyp files,
 101       # so that build/gyp_chromium can work.
 102       for nonessential_dir in (NONESSENTIAL_DIRS + TESTDIRS):
 103         dir_path = os.path.join(GetSourceDirectory(), nonessential_dir)
 104         if (name.startswith(dir_path) and
 105             os.path.isfile(name) and
 106             'gyp' not in name):
 107           return
 108
 109     tarfile.TarFile.add(self, name, arcname=arcname, recursive=recursive)
 110
 111
 112 def main(argv):
 113   parser = optparse.OptionParser()
 114   parser.add_option("--basename")
 115   parser.add_option("--remove-nonessential-files",
 116                     dest="remove_nonessential_files",
 117                     action="store_true", default=False)
 118   parser.add_option("--test-data", action="store_true")
 119   # TODO(phajdan.jr): Remove --xz option when it's not needed for compatibility.
 120   parser.add_option("--xz", action="store_true")
 121
 122   options, args = parser.parse_args(argv)
 123
 124   if len(args) != 1:
 125     print 'You must provide only one argument: output file name'
 126     print '(without .tar.xz extension).'
 127     return 1
 128
 129   if not os.path.exists(GetSourceDirectory()):
 130     print 'Cannot find the src directory ' + GetSourceDirectory()
 131     return 1
 132
 133   # These two commands are from src/DEPS; please keep them in sync.
 134   if subprocess.call(['python', 'build/util/lastchange.py', '-o',
 135                       'build/util/LASTCHANGE'], cwd=GetSourceDirectory()) != 0:
 136     print 'Could not run build/util/lastchange.py to update LASTCHANGE.'
 137     return 1
 138   if subprocess.call(['python', 'build/util/lastchange.py', '-s',
 139                       'third_party/WebKit', '-o',
 140                       'build/util/LASTCHANGE.blink'],
 141                      cwd=GetSourceDirectory()) != 0:
 142     print 'Could not run build/util/lastchange.py to update LASTCHANGE.blink.'
 143     return 1
 144
 145   output_fullname = args[0] + '.tar'
 146   output_basename = options.basename or os.path.basename(args[0])
 147
 148   archive = MyTarFile.open(output_fullname, 'w')
 149   archive.set_remove_nonessential_files(options.remove_nonessential_files)
 150   try:
 151     if options.test_data:
 152       for directory in TESTDIRS:
 153         archive.add(os.path.join(GetSourceDirectory(), directory),
 154                     arcname=os.path.join(output_basename, directory))
 155     else:
 156       archive.add(GetSourceDirectory(), arcname=output_basename)
 157   finally:
 158     archive.close()
 159
 160   if subprocess.call(['xz', '-9', output_fullname]) != 0:
 161     print 'xz -9 failed!'
 162     return 1
 163
 164   return 0
 165
 166
 167 if __name__ == "__main__":
 168   sys.exit(main(sys.argv[1:]))