intl/icu_sources_data.py

   1 #!/usr/bin/env python
   2 #
   3 # Any copyright is dedicated to the Public Domain.
   4 # http://creativecommons.org/publicdomain/zero/1.0/
   5 #
   6 # Generate SOURCES in sources.mozbuild files from ICU's Makefile.in
   7 # files, and also build a standalone copy of ICU using its build
   8 # system to generate a new copy of the in-tree ICU data file.
   9 #
  10 # This script expects to be run from `update-icu.sh` after the in-tree
  11 # copy of ICU has been updated.
  12
  13 from __future__ import print_function
  14
  15 import glob
  16 import os
  17 import shutil
  18 import subprocess
  19 import sys
  20 import tempfile
  21
  22 from mozpack import path as mozpath
  23
  24
  25 def find_source_file(dir, filename):
  26     base = os.path.splitext(filename)[0]
  27     for ext in ('.cpp', '.c'):
  28         f = mozpath.join(dir, base + ext)
  29         if os.path.isfile(f):
  30             return f
  31     raise Exception("Couldn't find source file for: %s" % filename)
  32
  33
  34 def get_sources_from_makefile(makefile):
  35     import pymake.parser
  36     from pymake.parserdata import SetVariable
  37     srcdir = os.path.dirname(makefile)
  38     for statement in pymake.parser.parsefile(makefile):
  39         if (isinstance(statement, SetVariable) and
  40                 statement.vnameexp.is_static_string and
  41                 statement.vnameexp.s == 'OBJECTS'):
  42             return sorted((find_source_file(srcdir, s)
  43                            for s in statement.value.split()),
  44                           key=lambda x: x.lower())
  45
  46
  47 def list_headers(path):
  48     result = []
  49     for name in os.listdir(path):
  50         f = mozpath.join(path, name)
  51         if os.path.isfile(f):
  52             result.append(f)
  53     return sorted(result, key=lambda x: x.lower())
  54
  55
  56 def write_sources(mozbuild, sources, headers):
  57     with open(mozbuild, 'wb') as f:
  58         f.write('# THIS FILE IS GENERATED BY /intl/icu_sources_data.py ' +
  59                 'DO NOT EDIT\n' +
  60                 'SOURCES += [\n')
  61         f.write(''.join("   '/%s',\n" % s for s in sources))
  62         f.write(']\n\n')
  63         f.write('EXPORTS.unicode += [\n')
  64         f.write(''.join("   '/%s',\n" % s for s in headers))
  65         f.write(']\n')
  66
  67
  68 def update_sources(topsrcdir):
  69     print('Updating ICU sources lists...')
  70     sys.path.append(mozpath.join(topsrcdir, 'build/pymake'))
  71     for d in ['common', 'i18n']:
  72         base_path = mozpath.join(topsrcdir, 'intl/icu/source/%s' % d)
  73         makefile = mozpath.join(base_path, 'Makefile.in')
  74         mozbuild = mozpath.join(topsrcdir,
  75                                 'config/external/icu/%s/sources.mozbuild' % d)
  76         sources = [mozpath.relpath(s, topsrcdir)
  77                    for s in get_sources_from_makefile(makefile)]
  78         headers = [mozpath.normsep(os.path.relpath(s, topsrcdir))
  79                    for s in list_headers(mozpath.join(base_path, 'unicode'))]
  80         write_sources(mozbuild, sources, headers)
  81
  82
  83 def try_run(name, command, cwd=None, **kwargs):
  84     try:
  85         with tempfile.NamedTemporaryFile(prefix=name, delete=False) as f:
  86             subprocess.check_call(command, cwd=cwd, stdout=f,
  87                                 stderr=subprocess.STDOUT, **kwargs)
  88     except subprocess.CalledProcessError:
  89         print('''Error running "{}" in directory {}
  90     See output in {}'''.format(' '.join(command), cwd, f.name),
  91             file=sys.stderr)
  92         return False
  93     else:
  94         os.unlink(f.name)
  95         return True
  96
  97
  98 def get_data_file(data_dir):
  99     files = glob.glob(mozpath.join(data_dir, 'icudt*.dat'))
 100     return files[0] if files else None
 101
 102
 103 def update_data_file(topsrcdir):
 104     objdir = tempfile.mkdtemp(prefix='icu-obj-')
 105     configure = mozpath.join(topsrcdir, 'intl/icu/source/configure')
 106     env = dict(os.environ)
 107     # bug 1262101 - these should be shared with the moz.build files
 108     env.update({
 109         'CPPFLAGS': ('-DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 ' +
 110                      '-DUCONFIG_NO_LEGACY_CONVERSION ' +
 111                      '-DUCONFIG_NO_TRANSLITERATION ' +
 112                      '-DUCONFIG_NO_REGULAR_EXPRESSIONS ' +
 113                      '-DUCONFIG_NO_BREAK_ITERATION ' +
 114                      '-DU_CHARSET_IS_UTF8')
 115     })
 116     print('Running ICU configure...')
 117     if not try_run(
 118             'icu-configure',
 119             ['sh', configure,
 120              '--with-data-packaging=archive',
 121              '--enable-static',
 122              '--disable-shared',
 123              '--disable-extras',
 124              '--disable-icuio',
 125              '--disable-layout',
 126              '--disable-tests',
 127              '--disable-samples',
 128              '--disable-strict'],
 129             cwd=objdir,
 130             env=env):
 131         return False
 132     print('Running ICU make...')
 133     if not try_run('icu-make', ['make'], cwd=objdir):
 134         return False
 135     print('Copying ICU data file...')
 136     tree_data_path = mozpath.join(topsrcdir,
 137                                   'config/external/icu/data/')
 138     old_data_file = get_data_file(tree_data_path)
 139     if not old_data_file:
 140         print('Error: no ICU data file in %s' % tree_data_path,
 141               file=sys.stderr)
 142         return False
 143     new_data_file = get_data_file(mozpath.join(objdir, 'data/out'))
 144     if not new_data_file:
 145         print('Error: no ICU data in ICU objdir', file=sys.stderr)
 146         return False
 147     if os.path.basename(old_data_file) != os.path.basename(new_data_file):
 148         # Data file name has the major version number embedded.
 149         os.unlink(old_data_file)
 150     shutil.copy(new_data_file, tree_data_path)
 151     try:
 152         shutil.rmtree(objdir)
 153     except:
 154         print('Warning: failed to remove %s' % objdir, file=sys.stderr)
 155     return True
 156
 157
 158 def main():
 159     if len(sys.argv) != 2:
 160         print('Usage: icu_sources_data.py <mozilla topsrcdir>',
 161               file=sys.stderr)
 162         sys.exit(1)
 163
 164     topsrcdir = mozpath.abspath(sys.argv[1])
 165     update_sources(topsrcdir)
 166     if not update_data_file(topsrcdir):
 167         print('Error updating ICU data file', file=sys.stderr)
 168         sys.exit(1)
 169
 170
 171 if __name__ == '__main__':
 172     main()