intl/icu_sources_data.py

   1 #!/usr/bin/env python
   2 #
   3 # Any copyright is dedicated to the Public Domain.
   4 # http://creativecommons.org/publicdomain/zero/1.0/
   5 #
   6 # Generate SOURCES in sources.mozbuild files from ICU's Makefile.in
   7 # files, and also build a standalone copy of ICU using its build
   8 # system to generate a new copy of the in-tree ICU data file.
   9 #
  10 # This script expects to be run from `update-icu.sh` after the in-tree
  11 # copy of ICU has been updated.
  12
  13 from __future__ import absolute_import
  14 from __future__ import print_function
  15
  16 import glob
  17 import multiprocessing
  18 import os
  19 import sets
  20 import shutil
  21 import subprocess
  22 import sys
  23 import tempfile
  24
  25 from mozpack import path as mozpath
  26
  27 # The following files have been determined to be dead/unused by a
  28 # semi-automated analysis. You can just remove any of the files below
  29 # if you need them. However, files marked with a "Cluster" comment
  30 # can only be removed together, as they have (directional) dependencies.
  31 # If you want to rerun this analysis, contact :decoder.
  32 UNUSED_SOURCES = sets.Set([
  33     'intl/icu/source/common/bytestrieiterator.cpp',
  34     'intl/icu/source/common/cstr.cpp',
  35     'intl/icu/source/common/cwchar.cpp',
  36     'intl/icu/source/common/icudataver.cpp',
  37     'intl/icu/source/common/icuplug.cpp',
  38     'intl/icu/source/common/pluralmap.cpp',
  39     'intl/icu/source/common/ucat.cpp',
  40     'intl/icu/source/common/ucnv2022.cpp',
  41     'intl/icu/source/common/ucnv_ct.cpp',
  42     'intl/icu/source/common/ucnvdisp.cpp',
  43     'intl/icu/source/common/ucnv_ext.cpp',
  44     'intl/icu/source/common/ucnvhz.cpp',
  45     'intl/icu/source/common/ucnvisci.cpp',
  46     'intl/icu/source/common/ucnv_lmb.cpp',
  47     'intl/icu/source/common/ucnvmbcs.cpp',
  48     'intl/icu/source/common/uidna.cpp',
  49     'intl/icu/source/common/unorm.cpp',
  50     'intl/icu/source/common/usc_impl.cpp',
  51     'intl/icu/source/common/ustr_wcs.cpp',
  52     'intl/icu/source/common/util_props.cpp',
  53     'intl/icu/source/i18n/anytrans.cpp',
  54     'intl/icu/source/i18n/brktrans.cpp',
  55     'intl/icu/source/i18n/casetrn.cpp',
  56     'intl/icu/source/i18n/cpdtrans.cpp',
  57     'intl/icu/source/i18n/esctrn.cpp',
  58     'intl/icu/source/i18n/fmtable_cnv.cpp',
  59     'intl/icu/source/i18n/funcrepl.cpp',
  60     'intl/icu/source/i18n/gender.cpp',
  61     'intl/icu/source/i18n/name2uni.cpp',
  62     'intl/icu/source/i18n/nortrans.cpp',
  63     'intl/icu/source/i18n/nultrans.cpp',
  64     'intl/icu/source/i18n/quant.cpp',
  65     'intl/icu/source/i18n/rbt.cpp',
  66     'intl/icu/source/i18n/rbt_data.cpp',
  67     'intl/icu/source/i18n/rbt_pars.cpp',
  68     'intl/icu/source/i18n/rbt_rule.cpp',
  69     'intl/icu/source/i18n/rbt_set.cpp',
  70     'intl/icu/source/i18n/regexcmp.cpp',
  71     'intl/icu/source/i18n/regeximp.cpp',
  72     'intl/icu/source/i18n/regexst.cpp',
  73     'intl/icu/source/i18n/regextxt.cpp',
  74     'intl/icu/source/i18n/rematch.cpp',
  75     'intl/icu/source/i18n/remtrans.cpp',
  76     'intl/icu/source/i18n/repattrn.cpp',
  77     'intl/icu/source/i18n/scientificnumberformatter.cpp',
  78     'intl/icu/source/i18n/strmatch.cpp',
  79     'intl/icu/source/i18n/strrepl.cpp',
  80     'intl/icu/source/i18n/titletrn.cpp',
  81     'intl/icu/source/i18n/tolowtrn.cpp',
  82     'intl/icu/source/i18n/toupptrn.cpp',
  83     'intl/icu/source/i18n/translit.cpp',
  84     'intl/icu/source/i18n/transreg.cpp',
  85     'intl/icu/source/i18n/tridpars.cpp',
  86     'intl/icu/source/i18n/udateintervalformat.cpp',
  87     'intl/icu/source/i18n/unesctrn.cpp',
  88     'intl/icu/source/i18n/uni2name.cpp',
  89     'intl/icu/source/i18n/uregexc.cpp',
  90     'intl/icu/source/i18n/uregex.cpp',
  91     'intl/icu/source/i18n/uregion.cpp',
  92     'intl/icu/source/i18n/uspoof_build.cpp',
  93     'intl/icu/source/i18n/uspoof_conf.cpp',
  94     'intl/icu/source/i18n/utrans.cpp',
  95     'intl/icu/source/i18n/vzone.cpp',
  96     'intl/icu/source/i18n/zrule.cpp',
  97     'intl/icu/source/i18n/ztrans.cpp',
  98
  99     # Cluster
 100     'intl/icu/source/common/resbund_cnv.cpp',
 101     'intl/icu/source/common/ures_cnv.cpp',
 102
 103     # Cluster
 104     'intl/icu/source/common/propsvec.cpp',
 105     'intl/icu/source/common/ucnvsel.cpp',
 106     'intl/icu/source/common/ucnv_set.cpp',
 107
 108     # Cluster
 109     'intl/icu/source/common/ubiditransform.cpp',
 110     'intl/icu/source/common/ushape.cpp',
 111
 112     # Cluster
 113     'intl/icu/source/i18n/csdetect.cpp',
 114     'intl/icu/source/i18n/csmatch.cpp',
 115     'intl/icu/source/i18n/csr2022.cpp',
 116     'intl/icu/source/i18n/csrecog.cpp',
 117     'intl/icu/source/i18n/csrmbcs.cpp',
 118     'intl/icu/source/i18n/csrsbcs.cpp',
 119     'intl/icu/source/i18n/csrucode.cpp',
 120     'intl/icu/source/i18n/csrutf8.cpp',
 121     'intl/icu/source/i18n/inputext.cpp',
 122     'intl/icu/source/i18n/ucsdet.cpp',
 123
 124     # Cluster
 125     'intl/icu/source/i18n/alphaindex.cpp',
 126     'intl/icu/source/i18n/ulocdata.cpp',
 127 ])
 128
 129
 130 def find_source_file(dir, filename):
 131     base = os.path.splitext(filename)[0]
 132     for ext in ('.cpp', '.c'):
 133         f = mozpath.join(dir, base + ext)
 134         if os.path.isfile(f):
 135             return f
 136     raise Exception("Couldn't find source file for: %s" % filename)
 137
 138
 139 def get_sources_from_makefile(makefile):
 140     import pymake.parser
 141     from pymake.parserdata import SetVariable
 142     srcdir = os.path.dirname(makefile)
 143     for statement in pymake.parser.parsefile(makefile):
 144         if (isinstance(statement, SetVariable) and
 145                 statement.vnameexp.is_static_string and
 146                 statement.vnameexp.s == 'OBJECTS'):
 147             return sorted((find_source_file(srcdir, s)
 148                            for s in statement.value.split()),
 149                           key=lambda x: x.lower())
 150
 151
 152 def list_headers(path):
 153     result = []
 154     for name in os.listdir(path):
 155         f = mozpath.join(path, name)
 156         if os.path.isfile(f):
 157             result.append(f)
 158     return sorted(result, key=lambda x: x.lower())
 159
 160
 161 def write_sources(mozbuild, sources, headers):
 162     with open(mozbuild, 'wb') as f:
 163         f.write('# THIS FILE IS GENERATED BY /intl/icu_sources_data.py ' +
 164                 'DO NOT EDIT\n' +
 165                 'SOURCES += [\n')
 166         f.write(''.join("   '/%s',\n" % s for s in sources))
 167         f.write(']\n\n')
 168         f.write('EXPORTS.unicode += [\n')
 169         f.write(''.join("   '/%s',\n" % s for s in headers))
 170         f.write(']\n')
 171
 172
 173 def update_sources(topsrcdir):
 174     print('Updating ICU sources lists...')
 175     sys.path.append(mozpath.join(topsrcdir, 'build/pymake'))
 176     for d in ['common', 'i18n']:
 177         base_path = mozpath.join(topsrcdir, 'intl/icu/source/%s' % d)
 178         makefile = mozpath.join(base_path, 'Makefile.in')
 179         mozbuild = mozpath.join(topsrcdir,
 180                                 'config/external/icu/%s/sources.mozbuild' % d)
 181         sources = [mozpath.relpath(s, topsrcdir)
 182                    for s in get_sources_from_makefile(makefile)]
 183         sources = filter(lambda x: x not in UNUSED_SOURCES, sources)
 184         headers = [mozpath.normsep(os.path.relpath(s, topsrcdir))
 185                    for s in list_headers(mozpath.join(base_path, 'unicode'))]
 186         write_sources(mozbuild, sources, headers)
 187
 188
 189 def try_run(name, command, cwd=None, **kwargs):
 190     try:
 191         with tempfile.NamedTemporaryFile(prefix=name, delete=False) as f:
 192             subprocess.check_call(command, cwd=cwd, stdout=f,
 193                                   stderr=subprocess.STDOUT, **kwargs)
 194     except subprocess.CalledProcessError:
 195         print('''Error running "{}" in directory {}
 196     See output in {}'''.format(' '.join(command), cwd, f.name),
 197               file=sys.stderr)
 198         return False
 199     else:
 200         os.unlink(f.name)
 201         return True
 202
 203
 204 def get_data_file(data_dir):
 205     files = glob.glob(mozpath.join(data_dir, 'icudt*.dat'))
 206     return files[0] if files else None
 207
 208
 209 def update_data_file(topsrcdir):
 210     objdir = tempfile.mkdtemp(prefix='icu-obj-')
 211     configure = mozpath.join(topsrcdir, 'intl/icu/source/configure')
 212     env = dict(os.environ)
 213     # bug 1262101 - these should be shared with the moz.build files
 214     env.update({
 215         'CPPFLAGS': ('-DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1 ' +
 216                      '-DU_HIDE_OBSOLETE_UTF_OLD_H=1' +
 217                      '-DUCONFIG_NO_LEGACY_CONVERSION ' +
 218                      '-DUCONFIG_NO_TRANSLITERATION ' +
 219                      '-DUCONFIG_NO_REGULAR_EXPRESSIONS ' +
 220                      '-DUCONFIG_NO_BREAK_ITERATION ' +
 221                      '-DU_CHARSET_IS_UTF8')
 222     })
 223
 224     # Exclude data that we currently don't need.
 225     #
 226     # The file format for ICU's data build tool is described at
 227     # <https://github.com/unicode-org/icu/blob/master/docs/userguide/icu_data/buildtool.md>.
 228     env["ICU_DATA_FILTER_FILE"] = mozpath.join(topsrcdir, 'intl/icu/data_filter.json')
 229
 230     print('Running ICU configure...')
 231     if not try_run(
 232             'icu-configure',
 233             ['sh', configure,
 234              '--with-data-packaging=archive',
 235              '--enable-static',
 236              '--disable-shared',
 237              '--disable-extras',
 238              '--disable-icuio',
 239              '--disable-layout',
 240              '--disable-layoutex',
 241              '--disable-tests',
 242              '--disable-samples',
 243              '--disable-strict'],
 244             cwd=objdir,
 245             env=env):
 246         return False
 247     print('Running ICU make...')
 248     if not try_run(
 249             'icu-make',
 250             ['make',
 251              '--jobs=%d' % multiprocessing.cpu_count(),
 252              '--output-sync'],
 253             cwd=objdir):
 254         return False
 255     print('Copying ICU data file...')
 256     tree_data_path = mozpath.join(topsrcdir,
 257                                   'config/external/icu/data/')
 258     old_data_file = get_data_file(tree_data_path)
 259     if not old_data_file:
 260         print('Error: no ICU data file in %s' % tree_data_path,
 261               file=sys.stderr)
 262         return False
 263     new_data_file = get_data_file(mozpath.join(objdir, 'data/out'))
 264     if not new_data_file:
 265         print('Error: no ICU data in ICU objdir', file=sys.stderr)
 266         return False
 267     if os.path.basename(old_data_file) != os.path.basename(new_data_file):
 268         # Data file name has the major version number embedded.
 269         os.unlink(old_data_file)
 270     shutil.copy(new_data_file, tree_data_path)
 271     try:
 272         shutil.rmtree(objdir)
 273     except Exception:
 274         print('Warning: failed to remove %s' % objdir, file=sys.stderr)
 275     return True
 276
 277
 278 def main():
 279     if len(sys.argv) != 2:
 280         print('Usage: icu_sources_data.py <mozilla topsrcdir>',
 281               file=sys.stderr)
 282         sys.exit(1)
 283
 284     topsrcdir = mozpath.abspath(sys.argv[1])
 285     update_sources(topsrcdir)
 286     if not update_data_file(topsrcdir):
 287         print('Error updating ICU data file', file=sys.stderr)
 288         sys.exit(1)
 289
 290
 291 if __name__ == '__main__':
 292     main()